From ef521be6438423142a6b65a635316ef7c549faa7 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Fri, 18 Apr 2025 14:51:03 -0600 Subject: [PATCH 01/17] DON'T SQUASH: Extract RPC failover functional tests --- .../block-hash-in-response.ts | 2089 +------------- .../tests/provider-api-tests/block-param.ts | 2485 +---------------- .../tests/provider-api-tests/helpers.ts | 20 +- .../provider-api-tests/no-block-param.ts | 2089 +------------- .../tests/provider-api-tests/rpc-failover.ts | 430 +++ 5 files changed, 796 insertions(+), 6317 deletions(-) create mode 100644 packages/network-controller/tests/provider-api-tests/rpc-failover.ts diff --git a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts index 33ab030fff8..3bd7ed88736 100644 --- a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts +++ b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts @@ -1,4 +1,3 @@ -import { ConstantBackoff } from '@metamask/controller-utils'; import { rpcErrors } from '@metamask/rpc-errors'; import type { ProviderType } from './helpers'; @@ -7,9 +6,8 @@ import { withMockedCommunications, withNetworkClient, } from './helpers'; -import { ignoreRejection } from '../../../../tests/helpers'; import { NetworkClientType } from '../../src/types'; -import { buildRootMessenger } from '../helpers'; +import { testsForRpcFailoverBehavior } from './rpc-failover'; type TestsForRpcMethodThatCheckForBlockHashInResponseOptions = { providerType: ProviderType; @@ -367,305 +365,24 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }, ); @@ -699,294 +416,24 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }); @@ -1064,360 +511,24 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }, ); @@ -1425,13 +536,14 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( describe.each(['ETIMEDOUT', 'ECONNRESET'])( 'if a %s error is thrown while making the request', (errorCode) => { + const error = new Error(errorCode); + // @ts-expect-error `code` does not exist on the Error type, but is + // still used by Node. + error.code = errorCode; + it('retries the request up to 5 times until it is successful', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -1469,10 +581,6 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( it('re-throws the error if it persists after 5 retries', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -1497,363 +605,22 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${errorCode}`, + }), }); }, ); @@ -1929,349 +696,33 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + body: 'invalid JSON', + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }); describe('if making the request throws a connection error', () => { + const error = new TypeError('Failed to fetch'); + it('retries the request up to 5 times until there is no connection error', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -2309,7 +760,6 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( it('re-throws the error if it persists after 5 retries', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -2334,337 +784,22 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: Failed to fetch`, - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: Failed to fetch`, - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${error.message}`, + }), }); }); } diff --git a/packages/network-controller/tests/provider-api-tests/block-param.ts b/packages/network-controller/tests/provider-api-tests/block-param.ts index 2848494c636..038bdaa1ead 100644 --- a/packages/network-controller/tests/provider-api-tests/block-param.ts +++ b/packages/network-controller/tests/provider-api-tests/block-param.ts @@ -1,7 +1,7 @@ -import { ConstantBackoff } from '@metamask/controller-utils'; import { rpcErrors } from '@metamask/rpc-errors'; +import type { Hex } from '@metamask/utils'; -import type { ProviderType } from './helpers'; +import type { MockRequest, ProviderType } from './helpers'; import { buildMockParams, buildRequestWithReplacedBlockParam, @@ -9,9 +9,8 @@ import { withMockedCommunications, withNetworkClient, } from './helpers'; -import { ignoreRejection } from '../../../../tests/helpers'; +import { testsForRpcFailoverBehavior } from './rpc-failover'; import { NetworkClientType } from '../../src/types'; -import { buildRootMessenger } from '../helpers'; type TestsForRpcMethodSupportingBlockParam = { providerType: ProviderType; @@ -456,362 +455,27 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. Note that to test that failovers work, all - // we have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Baz': 'Qux', - }, - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }, ); @@ -844,9 +508,6 @@ export function testsForRpcMethodSupportingBlockParam( '0x100', ), response: { - id: 12345, - jsonrpc: '2.0', - error: 'some error', httpStatus, }, }); @@ -859,362 +520,27 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Baz': 'Qux', - }, - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }); @@ -1322,418 +648,27 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. Note that to test that failovers work, all - // we have to do is make this request fail. - // TODO: We should be able to mock the request itself and not - // the block tracker request, but cannot because of a bug in - // eth-block-tracker. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. Note that to test that failovers work, all - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - error: 'Some error', - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: { + httpStatus, + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }, ); @@ -1741,16 +676,17 @@ export function testsForRpcMethodSupportingBlockParam( describe.each(['ETIMEDOUT', 'ECONNRESET'])( 'if a %s error is thrown while making the request', (errorCode) => { + const error = new Error(errorCode); + // @ts-expect-error `code` does not exist on the Error type, but is + // still used by Node. + error.code = errorCode; + it('retries the request up to 5 times until it is successful', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method, params: buildMockParams({ blockParam, blockParamIndex }), }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the // block-cache middleware will request the latest block number @@ -1806,10 +742,6 @@ export function testsForRpcMethodSupportingBlockParam( method, params: buildMockParams({ blockParam, blockParamIndex }), }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the // block-cache middleware will request the latest block number @@ -1846,416 +778,25 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, - // but is still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. Note that to test that failovers work, all - // we have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to - // make the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, - // but is still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, - // but is still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, - // but is still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${errorCode}`, + }), }); }, ); @@ -2362,421 +903,39 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we have - // to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - body: 'invalid JSON', - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: { + body: 'invalid JSON', + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }); describe('if making the request throws a connection error', () => { + const error = new TypeError('Failed to fetch'); + it('retries the request up to 5 times until there is no connection error', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method, params: buildMockParams({ blockParam, blockParamIndex }), }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the // block-cache middleware will request the latest block number @@ -2832,7 +991,6 @@ export function testsForRpcMethodSupportingBlockParam( method, params: buildMockParams({ blockParam, blockParamIndex }), }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the // block-cache middleware will request the latest block number @@ -2867,404 +1025,25 @@ export function testsForRpcMethodSupportingBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: Failed to fetch`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: Failed to fetch`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { - method, - params: buildMockParams({ blockParam, blockParamIndex }), - }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber: '0x100', - }); - primaryComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: buildRequestWithReplacedBlockParam( - request, - blockParamIndex, - '0x100', - ), - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: buildMockParams({ blockParam, blockParamIndex }), + }, + getRequestToMock: (request: MockRequest, blockNumber: Hex) => { + return buildRequestWithReplacedBlockParam( + request, + blockParamIndex, + blockNumber, + ); + }, + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${error.message}`, + }), }); }); }); diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 59bd53097c2..9b3e1fad208 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -73,7 +73,7 @@ function buildScopeForMockingRequests( // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any -type Request = { method: string; params?: any[] }; +export type MockRequest = { method: string; params?: any[] }; type Response = { id?: number | string; jsonrpc?: '2.0'; @@ -85,11 +85,11 @@ type Response = { result?: any; httpStatus?: number; }; -type BodyOrResponse = { body: JSONRPCResponse | string } | Response; +export type MockResponse = { body: JSONRPCResponse | string } | Response; type CurriedMockRpcCallOptions = { - request: Request; + request: MockRequest; // The response data. - response?: BodyOrResponse; + response?: MockResponse; /** * An error to throw while making the request. * Takes precedence over `response`. @@ -285,7 +285,7 @@ async function mockAllBlockTrackerRequests({ * response if it is successful or rejects with the error from the JSON-RPC * response otherwise. */ -function makeRpcCall(ethQuery: EthQuery, request: Request) { +function makeRpcCall(ethQuery: EthQuery, request: MockRequest) { return new Promise((resolve, reject) => { debug('[makeRpcCall] making request', request); // TODO: Replace `any` with type @@ -393,10 +393,10 @@ type MockNetworkClient = { clock: sinon.SinonFakeTimers; // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any - makeRpcCall: (request: Request) => Promise; + makeRpcCall: (request: MockRequest) => Promise; // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any - makeRpcCallsInSeries: (requests: Request[]) => Promise; + makeRpcCallsInSeries: (requests: MockRequest[]) => Promise; messenger: RootMessenger; chainId: Hex; rpcUrl: string; @@ -545,9 +545,9 @@ export async function withNetworkClient( const { provider, blockTracker } = networkClient; const ethQuery = new EthQuery(provider); - const curriedMakeRpcCall = (request: Request) => + const curriedMakeRpcCall = (request: MockRequest) => makeRpcCall(ethQuery, request); - const makeRpcCallsInSeries = async (requests: Request[]) => { + const makeRpcCallsInSeries = async (requests: MockRequest[]) => { const responses = []; for (const request of requests) { responses.push(await curriedMakeRpcCall(request)); @@ -621,7 +621,7 @@ export function buildMockParams({ * @returns The updated request object. */ export function buildRequestWithReplacedBlockParam( - { method, params = [] }: Request, + { method, params = [] }: MockRequest, blockParamIndex: number, // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/network-controller/tests/provider-api-tests/no-block-param.ts b/packages/network-controller/tests/provider-api-tests/no-block-param.ts index a5900d51d06..ef8dd12d54e 100644 --- a/packages/network-controller/tests/provider-api-tests/no-block-param.ts +++ b/packages/network-controller/tests/provider-api-tests/no-block-param.ts @@ -1,4 +1,3 @@ -import { ConstantBackoff } from '@metamask/controller-utils'; import { rpcErrors } from '@metamask/rpc-errors'; import type { ProviderType } from './helpers'; @@ -7,9 +6,8 @@ import { withMockedCommunications, withNetworkClient, } from './helpers'; -import { ignoreRejection } from '../../../../tests/helpers'; +import { testsForRpcFailoverBehavior } from './rpc-failover'; import { NetworkClientType } from '../../src/types'; -import { buildRootMessenger } from '../helpers'; type TestsForRpcMethodAssumingNoBlockParamOptions = { providerType: ProviderType; @@ -323,305 +321,24 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint/', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }, ); @@ -653,294 +370,24 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId, rpcUrl }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 15, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - }, - async ({ makeRpcCall, chainId }) => { - for (let i = 0; i < 14; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < 15; i++) { - await ignoreRejection(makeRpcCall(request)); - } - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: errorMessage, - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < 5; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: false, + getExpectedError: () => + expect.objectContaining({ + message: errorMessage, + }), }); }); @@ -1018,360 +465,24 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - error: 'Some error', - httpStatus, - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + httpStatus, + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }, ); @@ -1379,13 +490,14 @@ export function testsForRpcMethodAssumingNoBlockParam( describe.each(['ETIMEDOUT', 'ECONNRESET'])( 'if a %s error is thrown while making the request', (errorCode) => { + const error = new Error(errorCode); + // @ts-expect-error `code` does not exist on the Error type, but is + // still used by Node. + error.code = errorCode; + it('retries the request up to 5 times until it is successful', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -1423,10 +535,6 @@ export function testsForRpcMethodAssumingNoBlockParam( it('re-throws the error if it persists after 5 retries', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -1451,363 +559,22 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: ${errorCode}`, - }), - }); - }, - ); - }, - ); - }, - ); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new Error(errorCode); - // @ts-expect-error `code` does not exist on the Error type, but is - // still used by Node. - error.code = errorCode; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }, - ); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${errorCode}`, + }), }); }, ); @@ -1883,349 +650,33 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: expect.stringContaining(errorMessage), - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - response: { - body: 'invalid JSON', - }, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: { + body: 'invalid JSON', + }, + isRetriableFailure: true, + getExpectedError: () => + expect.objectContaining({ + message: expect.stringContaining(errorMessage), + }), }); }); describe('if making the request throws a connection error', () => { + const error = new TypeError('Failed to fetch'); + it('retries the request up to 5 times until there is no connection error', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -2263,7 +714,6 @@ export function testsForRpcMethodAssumingNoBlockParam( it('re-throws the error if it persists after 5 retries', async () => { await withMockedCommunications({ providerType }, async (comms) => { const request = { method }; - const error = new TypeError('Failed to fetch'); // The first time a block-cacheable request is made, the latest block // number is retrieved through the block tracker first. It doesn't @@ -2288,337 +738,22 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - it('fails over to the provided alternate RPC endpoint after 15 unsuccessful attempts', async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block tracker - // first. Note that to test that failovers work, all we - // have to do is make this request fail. - primaryComms.mockRpcCall({ - request: { - method: 'eth_blockNumber', - params: [], - }, - error, - times: 15, - }); - failoverComms.mockNextBlockTrackerRequest(); - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - }, - async ({ makeRpcCall, clock }) => { - // The block tracker will keep trying to poll until the - // eth_blockNumber request works, so we only have to make - // the request once. - return await waitForPromiseToBeFulfilledAfterRunningAllTimers( - makeRpcCall(request), - clock, - ); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, chainId, clock, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - await ignoreRejection(makeRpcCall(request)); - await ignoreRejection(makeRpcCall(request)); - await makeRpcCall(request); - - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( - { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to ${rpcUrl} failed, reason: Failed to fetch`, - }), - }, - ); - }, - ); - }, - ); - }); - }); - - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 15, - }); - // The block-ref middleware will make the request as specified - // except that the block param is replaced with the latest - // block number. - failoverComms.mockRpcCall({ - request, - error, - times: 15, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest(); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint/'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // Exceed max retries on primary - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary again - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on primary for final time, fail over - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover - await ignoreRejection(makeRpcCall(request)); - // Exceed max retries on failover for final time - await ignoreRejection(makeRpcCall(request)); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: 'https://failover.endpoint/', - error: expect.objectContaining({ - message: `request to https://failover.endpoint/ failed, reason: Failed to fetch`, - }), - }); - }, - ); - }, - ); - }); - }); - - it('allows RPC service options to be customized', async () => { - const backoffDuration = 100; - - await withMockedCommunications({ providerType }, async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (failoverComms) => { - const request = { method }; - const error = new TypeError('Failed to fetch'); - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest(); - primaryComms.mockRpcCall({ - request, - error, - times: 6, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: 2, - maxConsecutiveFailures: 6, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - // There are a total of 3 attempts (2 retries), - // and we call this 2 times for a total of 6 failures - await ignoreRejection(makeRpcCall(request)); - return await makeRpcCall(request); - }, - ); - - expect(result).toBe('ok'); - }, - ); - }); + testsForRpcFailoverBehavior({ + providerType, + requestToCall: { + method, + params: [], + }, + getRequestToMock: () => ({ + method, + params: [], + }), + failure: error, + isRetriableFailure: true, + getExpectedError: (url: string) => + expect.objectContaining({ + message: `request to ${url} failed, reason: ${error.message}`, + }), }); }); } diff --git a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts new file mode 100644 index 00000000000..199d68619cf --- /dev/null +++ b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts @@ -0,0 +1,430 @@ +import { ConstantBackoff } from '@metamask/controller-utils'; +import type { Hex } from '@metamask/utils'; + +import type { MockRequest, MockResponse, ProviderType } from './helpers'; +import { withMockedCommunications, withNetworkClient } from './helpers'; +import { ignoreRejection } from '../../../../tests/helpers'; +import { buildRootMessenger } from '../helpers'; + +/** + * Tests for RPC failover behavior. + * + * @param args - The arguments. + * @param args.providerType - The provider type. + * @param args.requestToCall - The request to call. + * @param args.getRequestToMock - Factory returning the request to mock. + * @param args.failure - The failure mock response to use. + * @param args.isRetriableFailure - Whether the failure gets retried. + * @param args.getExpectedError - Factory returning the expected error. + */ +export function testsForRpcFailoverBehavior({ + providerType, + requestToCall, + getRequestToMock, + failure, + isRetriableFailure, + getExpectedError, +}: { + providerType: ProviderType; + requestToCall: MockRequest; + getRequestToMock: (request: MockRequest, blockNumber: Hex) => MockRequest; + failure: MockResponse | Error | string; + isRetriableFailure: boolean; + getExpectedError: (url: string) => Error | jest.Constructable; +}) { + const blockNumber = '0x100'; + const backoffDuration = 100; + const maxConsecutiveFailures = 15; + const maxRetries = 4; + const numRequestsToMake = isRetriableFailure + ? maxConsecutiveFailures / (maxRetries + 1) + : maxConsecutiveFailures; + + describe('assuming RPC failover functionality is enabled', () => { + it(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { + await withMockedCommunications({ providerType }, async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + + const result = await withNetworkClient( + { + providerType, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); + }, + ); + + expect(result).toBe('ok'); + }, + ); + }); + }); + + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + + await withMockedCommunications({ providerType }, async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); + + await withNetworkClient( + { + providerType, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, chainId, rpcUrl }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + await makeRpcCall(request); + + expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( + { + chainId, + endpointUrl: rpcUrl, + failoverEndpointUrl, + error: getExpectedError(rpcUrl), + }, + ); + }, + ); + }, + ); + }); + }); + + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + + await withMockedCommunications({ providerType }, async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // Block tracker requests on the primary will fail over + failoverComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); + + await withNetworkClient( + { + providerType, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, chainId }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < maxConsecutiveFailures - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + for (let i = 0; i < maxConsecutiveFailures; i++) { + await ignoreRejection(makeRpcCall(request)); + } + + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenNthCalledWith(2, { + chainId, + endpointUrl: failoverEndpointUrl, + error: getExpectedError(failoverEndpointUrl), + }); + }, + ); + }, + ); + }); + }); + + it('allows RPC service options to be customized', async () => { + const customMaxConsecutiveFailures = 6; + const customMaxRetries = 2; + // This is okay. + // eslint-disable-next-line jest/no-conditional-in-test + const customNumRequestsToMake = isRetriableFailure + ? customMaxConsecutiveFailures / (customMaxRetries + 1) + : customMaxConsecutiveFailures; + + await withMockedCommunications( + { + providerType, + expectedHeaders: { + 'X-Foo': 'Bar', + }, + }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', + expectedHeaders: { + 'X-Baz': 'Qux', + }, + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: customMaxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + + const result = await withNetworkClient( + { + providerType, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: (rpcEndpointUrl) => { + const commonOptions = { fetch, btoa }; + // We need to return different results. + // eslint-disable-next-line jest/no-conditional-in-test + if (rpcEndpointUrl === 'https://failover.endpoint') { + const headers: HeadersInit = { + 'X-Baz': 'Qux', + }; + return { + ...commonOptions, + fetchOptions: { + headers, + }, + }; + } + const headers: HeadersInit = { + 'X-Foo': 'Bar', + }; + return { + ...commonOptions, + fetchOptions: { + headers, + }, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + maxRetries: customMaxRetries, + maxConsecutiveFailures: customMaxConsecutiveFailures, + }, + }; + }, + }, + async ({ makeRpcCall, clock }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < customNumRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); + }, + ); + + expect(result).toBe('ok'); + }, + ); + }, + ); + }); + }); +} From 825a3aa899fd1a5f3fd8839d4d8d201ac17af712 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Wed, 16 Apr 2025 11:34:21 -0600 Subject: [PATCH 02/17] WIP --- jest.config.packages.js | 2 +- packages/network-controller/CHANGELOG.md | 4 + packages/network-controller/package.json | 4 + .../src/NetworkController.ts | 55 +- ...create-auto-managed-network-client.test.ts | 827 +++++++++++++----- .../src/create-auto-managed-network-client.ts | 81 +- .../src/create-network-client.ts | 69 +- .../src/rpc-service/rpc-service-chain.test.ts | 114 +++ .../src/rpc-service/rpc-service-chain.ts | 21 +- .../tests/NetworkController.test.ts | 188 +++- packages/network-controller/tests/helpers.ts | 25 +- .../block-hash-in-response.ts | 4 +- .../tests/provider-api-tests/block-param.ts | 1 - .../tests/provider-api-tests/helpers.ts | 5 + .../tests/provider-api-tests/rpc-failover.ts | 76 +- .../tests/provider-api-tests/shared-tests.ts | 2 +- .../network-controller/tsconfig.build.json | 3 +- packages/network-controller/tsconfig.json | 3 + .../src/index.ts | 5 +- yarn.lock | 3 + 20 files changed, 1203 insertions(+), 289 deletions(-) diff --git a/jest.config.packages.js b/jest.config.packages.js index fd5e2eb5e94..1ec5510e276 100644 --- a/jest.config.packages.js +++ b/jest.config.packages.js @@ -180,7 +180,7 @@ module.exports = { // testRunner: "jest-circus/runner", // Default timeout of a test in milliseconds. - testTimeout: 30000, + testTimeout: 3000, // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href // testURL: "http://localhost", diff --git a/packages/network-controller/CHANGELOG.md b/packages/network-controller/CHANGELOG.md index 22c9414f03a..46d280e6eed 100644 --- a/packages/network-controller/CHANGELOG.md +++ b/packages/network-controller/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add optional `rpcFailoverEnabled` option to `NetworkController` constructor (`false` by default) ([#5668](https://github.com/MetaMask/core/pull/5668)) + ## [23.2.0] ### Added diff --git a/packages/network-controller/package.json b/packages/network-controller/package.json index 3c106d0b24c..d530fa1db8b 100644 --- a/packages/network-controller/package.json +++ b/packages/network-controller/package.json @@ -69,6 +69,7 @@ "devDependencies": { "@json-rpc-specification/meta-schema": "^1.0.6", "@metamask/auto-changelog": "^3.4.4", + "@metamask/remote-feature-flag-controller": "^1.6.0", "@types/jest": "^27.4.1", "@types/jest-when": "^2.7.3", "@types/lodash": "^4.14.191", @@ -85,6 +86,9 @@ "typedoc-plugin-missing-exports": "^2.0.0", "typescript": "~5.2.2" }, + "peerDependencies": { + "@metamask/remote-feature-flag-controller": "^1.5.0" + }, "engines": { "node": "^18.18 || >=20" }, diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index c8ab5d227f3..f39dce51eff 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -18,6 +18,7 @@ import { BuiltInNetworkName, } from '@metamask/controller-utils'; import EthQuery from '@metamask/eth-query'; +import type { RemoteFeatureFlagControllerStateChangeEvent } from '@metamask/remote-feature-flag-controller'; import { errorCodes } from '@metamask/rpc-errors'; import { createEventEmitterProxy } from '@metamask/swappable-obj-proxy'; import type { SwappableProxy } from '@metamask/swappable-obj-proxy'; @@ -497,6 +498,8 @@ export type NetworkControllerEvents = | NetworkControllerRpcEndpointDegradedEvent | NetworkControllerRpcEndpointRequestRetriedEvent; +export type AllowedEvents = RemoteFeatureFlagControllerStateChangeEvent; + export type NetworkControllerGetStateAction = ControllerGetStateAction< typeof controllerName, NetworkState @@ -589,12 +592,14 @@ export type NetworkControllerActions = | NetworkControllerRemoveNetworkAction | NetworkControllerUpdateNetworkAction; +export type AllowedActions = never; + export type NetworkControllerMessenger = RestrictedMessenger< typeof controllerName, - NetworkControllerActions, - NetworkControllerEvents, - never, - never + NetworkControllerActions | AllowedActions, + NetworkControllerEvents | AllowedEvents, + AllowedActions['type'], + AllowedEvents['type'] >; /** @@ -630,11 +635,15 @@ export type NetworkControllerOptions = { getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; - /** * An array of Hex Chain IDs representing the additional networks to be included as default. */ additionalDefaultNetworks?: AdditionalDefaultNetwork[]; + /** + * Whether or not requests sent to unavailable RPC endpoints should be + * automatically diverted to configured failover RPC endpoints. + */ + isRpcFailoverEnabled?: boolean; }; /** @@ -1085,6 +1094,11 @@ export class NetworkController extends BaseController< NetworkConfiguration >; + #isRpcFailoverEnabled: Exclude< + NetworkControllerOptions['isRpcFailoverEnabled'], + undefined + >; + /** * Constructs a NetworkController. * @@ -1098,6 +1112,7 @@ export class NetworkController extends BaseController< log, getRpcServiceOptions, additionalDefaultNetworks, + isRpcFailoverEnabled = false, } = options; const initialState = { ...getDefaultNetworkControllerState(additionalDefaultNetworks), @@ -1131,6 +1146,7 @@ export class NetworkController extends BaseController< this.#infuraProjectId = infuraProjectId; this.#log = log; this.#getRpcServiceOptions = getRpcServiceOptions; + this.#isRpcFailoverEnabled = isRpcFailoverEnabled; this.#previouslySelectedNetworkClientId = this.state.selectedNetworkClientId; @@ -1229,6 +1245,31 @@ export class NetworkController extends BaseController< ); } + enableRpcFailover() { + if (this.#isRpcFailoverEnabled) { + return; + } + + const networkClientsById = this.getNetworkClientRegistry(); + + for (const networkClient of Object.values(networkClientsById)) { + console.log('networkClient', networkClient.configuration); + if ( + networkClient.configuration.failoverRpcUrls && + networkClient.configuration.failoverRpcUrls.length > 0 + ) { + networkClient.enableRpcFailover(); + } + } + + this.#isRpcFailoverEnabled = true; + } + + disableRpcFailover() { + // Go through all RPC endpoints with failover RPC URLs defined and call + // .enableRpcFailover on the network clients + } + /** * Accesses the provider and block tracker for the currently selected network. * @returns The proxy and block tracker proxies. @@ -2639,6 +2680,7 @@ export class NetworkController extends BaseController< }, getRpcServiceOptions: this.#getRpcServiceOptions, messenger: this.messagingSystem, + isRpcFailoverEnabled: this.#isRpcFailoverEnabled, }); } else { autoManagedNetworkClientRegistry[NetworkClientType.Custom][ @@ -2653,6 +2695,7 @@ export class NetworkController extends BaseController< }, getRpcServiceOptions: this.#getRpcServiceOptions, messenger: this.messagingSystem, + isRpcFailoverEnabled: this.#isRpcFailoverEnabled, }); } } @@ -2813,6 +2856,7 @@ export class NetworkController extends BaseController< }, getRpcServiceOptions: this.#getRpcServiceOptions, messenger: this.messagingSystem, + isRpcFailoverEnabled: this.#isRpcFailoverEnabled, }), ] as const; } @@ -2828,6 +2872,7 @@ export class NetworkController extends BaseController< }, getRpcServiceOptions: this.#getRpcServiceOptions, messenger: this.messagingSystem, + isRpcFailoverEnabled: this.#isRpcFailoverEnabled, }), ] as const; }); diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index 4d9d6e8cceb..b89f0b5d9db 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -13,6 +13,10 @@ import type { } from './types'; import { NetworkClientType } from './types'; import { mockNetwork } from '../../../tests/mock-network'; +import { + buildCustomNetworkClientConfiguration, + buildFakeNetworkClient, +} from '../tests/helpers'; describe('createAutoManagedNetworkClient', () => { const networkClientConfigurations: [ @@ -45,6 +49,7 @@ describe('createAutoManagedNetworkClient', () => { btoa, }), messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, }); expect(configuration).toStrictEqual(networkClientConfiguration); @@ -60,6 +65,7 @@ describe('createAutoManagedNetworkClient', () => { btoa, }), messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, }); }).not.toThrow(); }); @@ -72,6 +78,7 @@ describe('createAutoManagedNetworkClient', () => { btoa, }), messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, }); // This also tests the `has` trap in the proxy @@ -95,98 +102,311 @@ describe('createAutoManagedNetworkClient', () => { expect('request' in provider).toBe(true); }); - it('returns a provider proxy that acts like a provider, forwarding requests to the network', async () => { - mockNetwork({ - networkClientConfiguration, - mocks: [ - { - request: { - method: 'test_method', - params: [], - }, - response: { - result: 'test response', + describe('when accessing the provider proxy', () => { + it('forwards requests to the network', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, }, - }, - ], + ], + }); + + const { provider } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + fetch, + btoa, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, + }); + + const result = await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + expect(result).toBe('test response'); }); - const { provider } = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - fetch, + it('creates the network client only once, even when the provider proxy is used to make requests multiple times', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + discardAfterMatching: false, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ btoa, - }), - messenger: getNetworkControllerMessenger(), + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const { provider } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + await provider.request({ + id: 2, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + expect(createNetworkClientMock).toHaveBeenCalledTimes(1); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); }); - const result = await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], + it('allows for enabling the RPC failover before the network client is initialized', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + discardAfterMatching: false, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); + autoManagedNetworkClient.enableRpcFailover(); + const { provider } = autoManagedNetworkClient; + + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); }); - expect(result).toBe('test response'); - }); - it('creates the network client only once, even when the provider proxy is used to make requests multiple times', async () => { - mockNetwork({ - networkClientConfiguration, - mocks: [ - { - request: { - method: 'test_method', - params: [], + it('allows for enabling the RPC failover after the network client is initialized', async () => { + const mockNetworkClient = buildFakeNetworkClient({ + configuration: networkClientConfiguration, + providerStubs: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, }, - response: { - result: 'test response', + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, }, - discardAfterMatching: false, - }, - ], - }); - const createNetworkClientMock = jest.spyOn( - createNetworkClientModule, - 'createNetworkClient', - ); - const getRpcServiceOptions = () => ({ - btoa, - fetch, - fetchOptions: { - headers: { - 'X-Foo': 'Bar', - }, - }, - policyOptions: { - maxRetries: 2, - maxConsecutiveFailures: 10, - }, - }); + ], + }); + const enableRpcFailoverMock = jest.spyOn( + mockNetworkClient, + 'enableRpcFailover', + ); + jest + .spyOn(createNetworkClientModule, 'createNetworkClient') + .mockReturnValue(mockNetworkClient); + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + btoa, + fetch, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, + }); + const { provider } = autoManagedNetworkClient; - const { provider } = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions, - messenger: getNetworkControllerMessenger(), - }); + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + autoManagedNetworkClient.enableRpcFailover(); + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); - await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], + expect(enableRpcFailoverMock).toHaveBeenCalled(); }); - await provider.request({ - id: 2, - jsonrpc: '2.0', - method: 'test_method', - params: [], + + it('allows for disabling the RPC failover before the network client is initialized', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + discardAfterMatching: false, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + autoManagedNetworkClient.disableRpcFailover(); + const { provider } = autoManagedNetworkClient; + + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); }); - expect(createNetworkClientMock).toHaveBeenCalledTimes(1); - expect(createNetworkClientMock).toHaveBeenCalledWith( - expect.objectContaining({ + + it('allows for disabling the RPC failover after the network client is initialized', async () => { + const mockNetworkClient = buildFakeNetworkClient({ configuration: networkClientConfiguration, - }), - ); + providerStubs: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + }, + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + }, + ], + }); + const disableRpcFailoverMock = jest.spyOn( + mockNetworkClient, + 'disableRpcFailover', + ); + jest + .spyOn(createNetworkClientModule, 'createNetworkClient') + .mockReturnValue(mockNetworkClient); + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + btoa, + fetch, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: true, + }); + const { provider } = autoManagedNetworkClient; + + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + autoManagedNetworkClient.disableRpcFailover(); + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + + expect(disableRpcFailoverMock).toHaveBeenCalled(); + }); }); it('returns a block tracker proxy that has the same interface as a block tracker', () => { @@ -197,6 +417,7 @@ describe('createAutoManagedNetworkClient', () => { btoa, }), messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, }); // This also tests the `has` trap in the proxy @@ -222,158 +443,352 @@ describe('createAutoManagedNetworkClient', () => { expect('checkForLatestBlock' in blockTracker).toBe(true); }); - it('returns a block tracker proxy that acts like a block tracker, exposing events to be listened to', async () => { - mockNetwork({ - networkClientConfiguration, - mocks: [ - { - request: { - method: 'eth_blockNumber', - params: [], + describe('when accessing the block tracker proxy', () => { + it('exposes events to be listened to', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, }, - response: { - result: '0x1', + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x2', + }, }, - }, - { - request: { - method: 'eth_blockNumber', - params: [], + ], + }); + + const { blockTracker } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + fetch, + btoa, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, + }); + + const blockNumberViaLatest = await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + expect(blockNumberViaLatest).toBe('0x1'); + const blockNumberViaSync = await new Promise((resolve) => { + blockTracker.once('sync', resolve); + }); + expect(blockNumberViaSync).toStrictEqual({ + oldBlock: '0x1', + newBlock: '0x2', + }); + }); + + it('creates the network client only once, even when the block tracker proxy is used multiple times', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, }, - response: { - result: '0x2', + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x2', + }, }, - }, - ], + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x3', + }, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const { blockTracker } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + await new Promise((resolve) => { + blockTracker.once('sync', resolve); + }); + await blockTracker.getLatestBlock(); + await blockTracker.checkForLatestBlock(); + expect(createNetworkClientMock).toHaveBeenCalledTimes(1); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); }); - const { blockTracker } = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - fetch, + it('allows for enabling the RPC failover before the network client is initialized', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ btoa, - }), - messenger: getNetworkControllerMessenger(), - }); + fetch, + }); + const messenger = getNetworkControllerMessenger(); - const blockNumberViaLatest = await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - expect(blockNumberViaLatest).toBe('0x1'); - const blockNumberViaSync = await new Promise((resolve) => { - blockTracker.once('sync', resolve); - }); - expect(blockNumberViaSync).toStrictEqual({ - oldBlock: '0x1', - newBlock: '0x2', + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); + autoManagedNetworkClient.enableRpcFailover(); + const { blockTracker } = autoManagedNetworkClient; + + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); }); - }); - it('creates the network client only once, even when the block tracker proxy is used multiple times', async () => { - mockNetwork({ - networkClientConfiguration, - mocks: [ - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x1', + it('allows for enabling the RPC failover after the network client is initialized', async () => { + const mockNetworkClient = buildFakeNetworkClient({ + configuration: networkClientConfiguration, + providerStubs: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, }, - }, - { - request: { - method: 'eth_blockNumber', - params: [], + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x2', + }, }, - response: { - result: '0x2', + ], + }); + const enableRpcFailoverMock = jest.spyOn( + mockNetworkClient, + 'enableRpcFailover', + ); + jest + .spyOn(createNetworkClientModule, 'createNetworkClient') + .mockReturnValue(mockNetworkClient); + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + btoa, + fetch, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, + }); + const { blockTracker } = autoManagedNetworkClient; + + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + autoManagedNetworkClient.enableRpcFailover(); + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + + expect(enableRpcFailoverMock).toHaveBeenCalled(); + }); + + it('allows for disabling the RPC failover before the network client is initialized', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, }, - }, - { - request: { - method: 'eth_blockNumber', - params: [], + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + autoManagedNetworkClient.disableRpcFailover(); + const { blockTracker } = autoManagedNetworkClient; + + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); + }); + + it('allows for disabling the RPC failover after the network client is initialized', async () => { + const mockNetworkClient = buildFakeNetworkClient({ + configuration: networkClientConfiguration, + providerStubs: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, }, - response: { - result: '0x3', + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x2', + }, }, - }, - ], - }); - const createNetworkClientMock = jest.spyOn( - createNetworkClientModule, - 'createNetworkClient', - ); - const getRpcServiceOptions = () => ({ - btoa, - fetch, - fetchOptions: { - headers: { - 'X-Foo': 'Bar', - }, - }, - policyOptions: { - maxRetries: 2, - maxConsecutiveFailures: 10, - }, - }); + ], + }); + const disableRpcFailoverMock = jest.spyOn( + mockNetworkClient, + 'disableRpcFailover', + ); + jest + .spyOn(createNetworkClientModule, 'createNetworkClient') + .mockReturnValue(mockNetworkClient); + const autoManagedNetworkClient = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + btoa, + fetch, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: true, + }); + const { blockTracker } = autoManagedNetworkClient; - const { blockTracker } = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions, - messenger: getNetworkControllerMessenger(), - }); + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + autoManagedNetworkClient.disableRpcFailover(); + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); - await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - await new Promise((resolve) => { - blockTracker.once('sync', resolve); + expect(disableRpcFailoverMock).toHaveBeenCalled(); }); - await blockTracker.getLatestBlock(); - await blockTracker.checkForLatestBlock(); - expect(createNetworkClientMock).toHaveBeenCalledTimes(1); - expect(createNetworkClientMock).toHaveBeenCalledWith( - expect.objectContaining({ - configuration: networkClientConfiguration, - }), - ); }); + }); - it('allows the block tracker to be destroyed', () => { - mockNetwork({ - networkClientConfiguration, - mocks: [ - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x1', - }, + it('destroys the block tracker when destroyed', () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], }, - ], - }); - const { blockTracker, destroy } = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - fetch, - btoa, - }), - messenger: getNetworkControllerMessenger(), - }); - // Start the block tracker - blockTracker.on('latest', () => { - // do nothing - }); + response: { + result: '0x1', + }, + }, + ], + }); + const { blockTracker, destroy } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions: () => ({ + fetch, + btoa, + }), + messenger: getNetworkControllerMessenger(), + isRpcFailoverEnabled: false, + }); + // Start the block tracker + blockTracker.on('latest', () => { + // do nothing + }); - destroy(); + destroy(); - expect(blockTracker.isRunning()).toBe(false); - }); + expect(blockTracker.isRunning()).toBe(false); }); } }); diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index 9fde14a2f5a..d027bf04030 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -40,6 +40,8 @@ export type AutoManagedNetworkClient< provider: ProxyWithAccessibleTarget; blockTracker: ProxyWithAccessibleTarget; destroy: () => void; + enableRpcFailover: () => void; + disableRpcFailover: () => void; }; /** @@ -67,6 +69,9 @@ const UNINITIALIZED_TARGET = { __UNINITIALIZED__: true }; * @param args.getRpcServiceOptions - Factory for constructing RPC service * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. * @param args.messenger - The network controller messenger. + * @param args.isRpcFailoverEnabled - Whether or not requests sent to the RPC + * endpoint for this network should be automatically diverted to failover RPC + * endpoints (if defined). * @returns The auto-managed network client. */ export function createAutoManagedNetworkClient< @@ -75,15 +80,35 @@ export function createAutoManagedNetworkClient< networkClientConfiguration, getRpcServiceOptions, messenger, + isRpcFailoverEnabled: initialRpcFailoverEnabled, }: { networkClientConfiguration: Configuration; getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; messenger: NetworkControllerMessenger; + isRpcFailoverEnabled: boolean; }): AutoManagedNetworkClient { + let isRpcFailoverEnabled = initialRpcFailoverEnabled; let networkClient: NetworkClient | undefined; + const ensureNetworkClientCreated = (): NetworkClient => { + networkClient ??= createNetworkClient({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled, + }); + + if (networkClient === undefined) { + throw new Error( + "It looks like `createNetworkClient` didn't return anything. Perhaps it's being mocked?", + ); + } + + return networkClient; + }; + const providerProxy = new Proxy(UNINITIALIZED_TARGET, { // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -92,17 +117,7 @@ export function createAutoManagedNetworkClient< return networkClient?.provider; } - networkClient ??= createNetworkClient({ - configuration: networkClientConfiguration, - getRpcServiceOptions, - messenger, - }); - if (networkClient === undefined) { - throw new Error( - "It looks like `createNetworkClient` didn't return anything. Perhaps it's being mocked?", - ); - } - const { provider } = networkClient; + const { provider } = ensureNetworkClientCreated(); if (propertyName in provider) { // Typecast: We know that `[propertyName]` is a propertyName on @@ -133,12 +148,7 @@ export function createAutoManagedNetworkClient< if (propertyName === REFLECTIVE_PROPERTY_NAME) { return true; } - networkClient ??= createNetworkClient({ - configuration: networkClientConfiguration, - getRpcServiceOptions, - messenger, - }); - const { provider } = networkClient; + const { provider } = ensureNetworkClientCreated(); return propertyName in provider; }, }); @@ -153,17 +163,7 @@ export function createAutoManagedNetworkClient< return networkClient?.blockTracker; } - networkClient ??= createNetworkClient({ - configuration: networkClientConfiguration, - getRpcServiceOptions, - messenger, - }); - if (networkClient === undefined) { - throw new Error( - "It looks like createNetworkClient returned undefined. Perhaps it's mocked?", - ); - } - const { blockTracker } = networkClient; + const { blockTracker } = ensureNetworkClientCreated(); if (propertyName in blockTracker) { // Typecast: We know that `[propertyName]` is a propertyName on @@ -194,12 +194,7 @@ export function createAutoManagedNetworkClient< if (propertyName === REFLECTIVE_PROPERTY_NAME) { return true; } - networkClient ??= createNetworkClient({ - configuration: networkClientConfiguration, - getRpcServiceOptions, - messenger, - }); - const { blockTracker } = networkClient; + const { blockTracker } = ensureNetworkClientCreated(); return propertyName in blockTracker; }, }, @@ -209,10 +204,28 @@ export function createAutoManagedNetworkClient< networkClient?.destroy(); }; + const enableRpcFailover = () => { + if (networkClient) { + networkClient.enableRpcFailover(); + } else { + isRpcFailoverEnabled = true; + } + }; + + const disableRpcFailover = () => { + if (networkClient) { + networkClient.disableRpcFailover(); + } else { + isRpcFailoverEnabled = false; + } + }; + return { configuration: networkClientConfiguration, provider: providerProxy, blockTracker: blockTrackerProxy, destroy, + enableRpcFailover, + disableRpcFailover, }; } diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index 604e94a02d7..23feb52b404 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -46,6 +46,8 @@ export type NetworkClient = { provider: Provider; blockTracker: BlockTracker; destroy: () => void; + enableRpcFailover: () => void; + disableRpcFailover: () => void; }; /** @@ -57,34 +59,43 @@ export type NetworkClient = { * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. * @param args.messenger - The network controller messenger. * See {@link NetworkControllerOptions.getRpcServiceOptions}. + * @param args.isRpcFailoverEnabled - Whether or not requests sent to the RPC + * endpoint for this network should be automatically diverted to failover RPC + * endpoints (if defined). * @returns The network client. */ export function createNetworkClient({ configuration, getRpcServiceOptions, messenger, + isRpcFailoverEnabled: initialRpcFailoverEnabled, }: { configuration: NetworkClientConfiguration; getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; messenger: NetworkControllerMessenger; + isRpcFailoverEnabled: boolean; }): NetworkClient { + let isRpcFailoverEnabled = initialRpcFailoverEnabled; const primaryEndpointUrl = configuration.type === NetworkClientType.Infura ? `https://${configuration.network}.infura.io/v3/${configuration.infuraProjectId}` : configuration.rpcUrl; - const availableEndpointUrls = [ - primaryEndpointUrl, - ...(configuration.failoverRpcUrls ?? []), - ]; - const rpcService = new RpcServiceChain( + const determineAvailableEndpointUrls = (givenRpcFailoverEnabled: boolean) => { + return givenRpcFailoverEnabled + ? [primaryEndpointUrl, ...(configuration.failoverRpcUrls ?? [])] + : [primaryEndpointUrl]; + }; + const availableEndpointUrls = + determineAvailableEndpointUrls(isRpcFailoverEnabled); + const rpcServiceChain = new RpcServiceChain( availableEndpointUrls.map((endpointUrl) => ({ ...getRpcServiceOptions(endpointUrl), endpointUrl, })), ); - rpcService.onBreak(({ endpointUrl, failoverEndpointUrl, ...rest }) => { + rpcServiceChain.onBreak(({ endpointUrl, failoverEndpointUrl, ...rest }) => { let error: unknown; if ('error' in rest) { error = rest.error; @@ -99,28 +110,57 @@ export function createNetworkClient({ error, }); }); - rpcService.onDegraded(({ endpointUrl }) => { + rpcServiceChain.onDegraded(({ endpointUrl }) => { messenger.publish('NetworkController:rpcEndpointDegraded', { chainId: configuration.chainId, endpointUrl, }); }); - rpcService.onRetry(({ endpointUrl, attempt }) => { + rpcServiceChain.onRetry(({ endpointUrl, attempt }) => { messenger.publish('NetworkController:rpcEndpointRequestRetried', { endpointUrl, attempt, }); }); + const updateRpcServices = () => { + rpcServiceChain.updateServices( + determineAvailableEndpointUrls(isRpcFailoverEnabled).map( + (endpointUrl) => ({ + ...getRpcServiceOptions(endpointUrl), + endpointUrl, + }), + ), + ); + }; + + const enableRpcFailover = () => { + if (isRpcFailoverEnabled) { + return; + } + + isRpcFailoverEnabled = true; + updateRpcServices(); + }; + + const disableRpcFailover = () => { + if (!isRpcFailoverEnabled) { + return; + } + + isRpcFailoverEnabled = false; + updateRpcServices(); + }; + const rpcApiMiddleware = configuration.type === NetworkClientType.Infura ? createInfuraMiddleware({ - rpcService, + rpcService: rpcServiceChain, options: { source: 'metamask', }, }) - : createFetchMiddleware({ rpcService }); + : createFetchMiddleware({ rpcService: rpcServiceChain }); const rpcProvider = providerFromMiddleware(rpcApiMiddleware); @@ -159,7 +199,14 @@ export function createNetworkClient({ blockTracker.destroy(); }; - return { configuration, provider, blockTracker, destroy }; + return { + configuration, + provider, + blockTracker, + destroy, + enableRpcFailover, + disableRpcFailover, + }; } /** diff --git a/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts b/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts index c4edfd921a7..301c9eb34d5 100644 --- a/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts +++ b/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts @@ -15,6 +15,120 @@ describe('RpcServiceChain', () => { clock.restore(); }); + describe('updateServices', () => { + it('replaces the underlying RPC services with a new set constructed from the given configuration objects', async () => { + nock('https://first.chain') + .post( + '/', + { + id: 1, + jsonrpc: '2.0', + method: 'eth_chainId', + params: [], + }, + { + reqheaders: {}, + }, + ) + .times(15) + .reply(503); + nock('https://second.chain') + .post('/', { + id: 1, + jsonrpc: '2.0', + method: 'eth_chainId', + params: [], + }) + .times(15) + .reply(503); + nock('https://third.chain') + .post('/', { + id: 1, + jsonrpc: '2.0', + method: 'eth_chainId', + params: [], + }) + .reply(200, { + id: 1, + jsonrpc: '2.0', + result: 'ok', + }); + + const rpcServiceChain = new RpcServiceChain([ + { + fetch, + btoa, + endpointUrl: 'https://some.other.chain', + }, + ]); + rpcServiceChain.updateServices([ + { + fetch, + btoa, + endpointUrl: 'https://first.chain', + }, + { + fetch, + btoa, + endpointUrl: 'https://second.chain', + fetchOptions: { + headers: { + 'X-Foo': 'Bar', + }, + }, + }, + { + fetch, + btoa, + endpointUrl: 'https://third.chain', + }, + ]); + rpcServiceChain.onRetry(() => { + // We don't need to await this promise; adding it to the promise + // queue is enough to continue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.nextAsync(); + }); + + const jsonRpcRequest = { + id: 1, + jsonrpc: '2.0' as const, + method: 'eth_chainId', + params: [], + }; + // Retry the first endpoint until max retries is hit. + await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( + 'Gateway timeout', + ); + // Retry the first endpoint again, until max retries is hit. + await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( + 'Gateway timeout', + ); + // Retry the first endpoint for a third time, until max retries is hit. + // The circuit will break on the last time, and the second endpoint will + // be retried, until max retries is hit. + await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( + 'Gateway timeout', + ); + // Try the first endpoint, see that the circuit is broken, and retry the + // second endpoint, until max retries is hit. + await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( + 'Gateway timeout', + ); + // Try the first endpoint, see that the circuit is broken, and retry the + // second endpoint, until max retries is hit. + // The circuit will break on the last time, and the third endpoint will + // be hit. This is finally a success. + const response = await rpcServiceChain.request(jsonRpcRequest); + + expect(response).toStrictEqual({ + id: 1, + jsonrpc: '2.0', + result: 'ok', + }); + }); + }); + describe('onRetry', () => { it('returns a listener which can be disposed', () => { const rpcServiceChain = new RpcServiceChain([ diff --git a/packages/network-controller/src/rpc-service/rpc-service-chain.ts b/packages/network-controller/src/rpc-service/rpc-service-chain.ts index 65921f27695..73595728cea 100644 --- a/packages/network-controller/src/rpc-service/rpc-service-chain.ts +++ b/packages/network-controller/src/rpc-service/rpc-service-chain.ts @@ -17,7 +17,7 @@ import type { FetchOptions } from './shared'; * failovers. */ export class RpcServiceChain implements RpcServiceRequestable { - readonly #services: RpcService[]; + #services: RpcService[] = []; /** * Constructs a new RpcServiceChain object. @@ -28,6 +28,25 @@ export class RpcServiceChain implements RpcServiceRequestable { */ constructor( rpcServiceConfigurations: Omit[], + ) { + this.updateServices(rpcServiceConfigurations); + } + + /** + * Replaces the underlying RPC services with a new set constructed from the + * given configuration objects. + * + * This is useful when toggling the RPC failover feature without needing to + * reconstruct entire network clients or RPC middleware stacks. (This is + * possible because the fetch and Infura middleware take this whole + * RpcServiceChain object, not individual RpcService instances.) + * + * @param rpcServiceConfigurations - The options for the RPC services + * that you want to construct. Each object in this array is the same as + * {@link RpcServiceOptions}. + */ + updateServices( + rpcServiceConfigurations: Omit[], ) { this.#services = this.#buildRpcServiceChain(rpcServiceConfigurations); } diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index e40de31fe2b..261cfc62bf2 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -16,11 +16,12 @@ import { import { rpcErrors } from '@metamask/rpc-errors'; import type { Hex } from '@metamask/utils'; import assert from 'assert'; -import type { Patch } from 'immer'; +import { produceWithPatches, type Patch } from 'immer'; import { when, resetAllWhenMocks } from 'jest-when'; import { inspect, isDeepStrictEqual, promisify } from 'util'; import { v4 as uuidV4 } from 'uuid'; +import type { RootMessenger } from './helpers'; import { buildAddNetworkCustomRpcEndpointFields, buildAddNetworkFields, @@ -42,6 +43,7 @@ import type { FakeProviderStub } from '../../../tests/fake-provider'; import { FakeProvider } from '../../../tests/fake-provider'; import { NetworkStatus } from '../src/constants'; import * as createAutoManagedNetworkClientModule from '../src/create-auto-managed-network-client'; +import type { AutoManagedNetworkClient } from '../src/create-auto-managed-network-client'; import type { NetworkClient } from '../src/create-network-client'; import { createNetworkClient } from '../src/create-network-client'; import type { @@ -639,6 +641,126 @@ describe('NetworkController', () => { }); }); + describe('enableRpcFailover', () => { + describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { + it.todo('does nothing'); + }); + + describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { + it.only('calls enableRpcFailover on only the network clients whose RPC endpoints have configured failover URLs', async () => { + await withController( + { + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'enableRpcFailover'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); + /* + const fakeAutoManagedNetworkClients = [ + { + enableRpcFailover: jest.fn(), + }, + { + enableRpcFailover: jest.fn(), + }, + { + enableRpcFailover: jest.fn(), + }, + ]; + createAutoManagedNetworkClientSpy.mockImplementation( + // @ts-expect-error We are intentionally returning partial + // AutoManagedCustomNetworkClients. + ({ networkClientConfiguration }) => { + if ( + networkClientConfiguration.type === NetworkClientType.Infura + ) { + return fakeAutoManagedNetworkClients[0]; + } + if ( + networkClientConfiguration.type === + NetworkClientType.Custom && + networkClientConfiguration.rpcUrl === 'https://test.network/1' + ) { + return fakeAutoManagedNetworkClients[1]; + } + if ( + networkClientConfiguration.type === + NetworkClientType.Custom && + networkClientConfiguration.rpcUrl === 'https://test.network/2' + ) { + return fakeAutoManagedNetworkClients[2]; + } + throw new Error( + `Unknown network client configuration ${JSON.stringify( + networkClientConfiguration, + )}`, + ); + }, + ); + */ + + controller.enableRpcFailover(); + + expect(autoManagedNetworkClients).toHaveLength(3); + for (const autoManagedNetworkClient of autoManagedNetworkClients) { + expect( + autoManagedNetworkClient.enableRpcFailover, + ).toHaveBeenCalled(); + } + }, + ); + }); + }); + }); + describe('destroy', () => { it('does not throw if called before the provider is initialized', async () => { await withController(async ({ controller }) => { @@ -1035,10 +1157,7 @@ describe('NetworkController', () => { { messenger, }: { - messenger: Messenger< - NetworkControllerActions, - NetworkControllerEvents - >; + messenger: RootMessenger; }, args: Parameters, ): ReturnType => @@ -2975,13 +3094,7 @@ describe('NetworkController', () => { ], [ 'NetworkController:getNetworkConfigurationByChainId', - ({ - messenger, - chainId, - }: { - messenger: Messenger; - chainId: Hex; - }) => + ({ messenger, chainId }: { messenger: RootMessenger; chainId: Hex }) => messenger.call( 'NetworkController:getNetworkConfigurationByChainId', chainId, @@ -3094,7 +3207,7 @@ describe('NetworkController', () => { messenger, networkClientId, }: { - messenger: Messenger; + messenger: RootMessenger; networkClientId: NetworkClientId; }) => messenger.call( @@ -3624,6 +3737,7 @@ describe('NetworkController', () => { }), infuraProjectId, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, ({ controller, networkControllerMessenger }) => { const defaultRpcEndpoint: InfuraRpcEndpoint = { @@ -3673,6 +3787,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); expect(createAutoManagedNetworkClientSpy).toHaveBeenNthCalledWith( @@ -3687,6 +3802,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); expect(createAutoManagedNetworkClientSpy).toHaveBeenNthCalledWith( @@ -3701,6 +3817,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); const networkConfigurationsByNetworkClientId = @@ -5061,13 +5178,12 @@ describe('NetworkController', () => { }, infuraProjectId, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { const infuraRpcEndpoint: InfuraRpcEndpoint = { failoverUrls: ['https://failover.endpoint'], networkClientId: infuraNetworkType, - // ESLint is mistaken here. - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions url: `https://${infuraNetworkType}.infura.io/v3/{infuraProjectId}`, type: RpcEndpointType.Infura, }; @@ -5094,6 +5210,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); const networkConfigurationsByNetworkClientId = @@ -5288,6 +5405,7 @@ describe('NetworkController', () => { }, infuraProjectId: 'some-infura-project-id', getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { const [rpcEndpoint1, rpcEndpoint2] = [ @@ -5321,6 +5439,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect( createAutoManagedNetworkClientSpy, @@ -5334,6 +5453,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); const networkConfigurationsByNetworkClientId = @@ -6272,6 +6392,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockReturnValue(buildFakeClient()); @@ -6299,6 +6420,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); const networkConfigurationsByNetworkClientId = getNetworkConfigurationsByNetworkClientId( @@ -7130,6 +7252,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { await controller.updateNetwork('0x1337', { @@ -7162,6 +7285,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); expect(createAutoManagedNetworkClientSpy).toHaveBeenNthCalledWith( @@ -7176,6 +7300,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); @@ -8114,6 +8239,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -8154,6 +8280,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect( getNetworkConfigurationsByNetworkClientId( @@ -9276,6 +9403,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -9311,6 +9439,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect( createAutoManagedNetworkClientSpy, @@ -9324,6 +9453,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); const networkConfigurationsByNetworkClientId = @@ -9984,6 +10114,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -10024,6 +10155,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect(createAutoManagedNetworkClientSpy).toHaveBeenCalledWith({ networkClientConfiguration: { @@ -10035,6 +10167,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect( @@ -10713,6 +10846,7 @@ describe('NetworkController', () => { }, infuraProjectId: 'some-infura-project-id', getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -10752,6 +10886,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); expect( createAutoManagedNetworkClientSpy, @@ -10765,6 +10900,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }); const networkConfigurationsByChainId = @@ -11407,6 +11543,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + isRpcFailoverEnabled: true, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation(({ configuration }) => { @@ -11440,6 +11577,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); expect(createAutoManagedNetworkClientSpy).toHaveBeenNthCalledWith( @@ -11454,6 +11592,7 @@ describe('NetworkController', () => { }, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled: true, }, ); @@ -14782,7 +14921,7 @@ type WithControllerCallback = ({ controller, }: { controller: NetworkController; - messenger: Messenger; + messenger: RootMessenger; networkControllerMessenger: NetworkControllerMessenger; }) => Promise | ReturnValue; @@ -14835,6 +14974,17 @@ async function withController( */ function buildFakeClient( provider: Provider = buildFakeProvider(), + { + enableRpcFailover = () => { + // do nothing + }, + disableRpcFailover = () => { + // do nothing + }, + }: { + enableRpcFailover?: () => void; + disableRpcFailover?: () => void; + } = {}, ): NetworkClient { return { configuration: { @@ -14849,6 +14999,8 @@ function buildFakeClient( destroy: () => { // do nothing }, + enableRpcFailover, + disableRpcFailover, }; } @@ -14961,7 +15113,7 @@ async function waitForPublishedEvents({ // do nothing }, }: { - messenger: Messenger; + messenger: RootMessenger; eventType: E['type']; count?: number; filter?: (payload: E['payload']) => boolean; @@ -15092,7 +15244,7 @@ async function waitForStateChanges({ operation, beforeResolving, }: { - messenger: Messenger; + messenger: RootMessenger; propertyPath?: string[]; count?: number; wait?: number; diff --git a/packages/network-controller/tests/helpers.ts b/packages/network-controller/tests/helpers.ts index 30def58c9ef..b5d5b4da3b4 100644 --- a/packages/network-controller/tests/helpers.ts +++ b/packages/network-controller/tests/helpers.ts @@ -25,6 +25,8 @@ import type { AutoManagedNetworkClient } from '../src/create-auto-managed-networ import type { AddNetworkCustomRpcEndpointFields, AddNetworkFields, + AllowedActions, + AllowedEvents, CustomRpcEndpoint, InfuraRpcEndpoint, NetworkControllerActions, @@ -40,8 +42,8 @@ import type { import { NetworkClientType } from '../src/types'; export type RootMessenger = Messenger< - NetworkControllerActions, - NetworkControllerEvents + NetworkControllerActions | AllowedActions, + NetworkControllerEvents | AllowedEvents >; /** @@ -73,7 +75,7 @@ export const TESTNET = { * @returns The messenger. */ export function buildRootMessenger(): RootMessenger { - return new Messenger(); + return new Messenger(); } /** @@ -88,7 +90,7 @@ export function buildNetworkControllerMessenger( return messenger.getRestricted({ name: 'NetworkController', allowedActions: [], - allowedEvents: [], + allowedEvents: ['RemoteFeatureFlagController:stateChange'], }); } @@ -100,14 +102,25 @@ export function buildNetworkControllerMessenger( * @param args.configuration - The desired network client configuration. * @param args.providerStubs - Objects that allow for stubbing specific provider * requests. + * @param args.enableRpcFailover - Override for the `enableRpcFailover` method. + * @param args.disableRpcFailover - Override for the `disableRpcFailover` + * method. * @returns The fake network client. */ -function buildFakeNetworkClient({ +export function buildFakeNetworkClient({ configuration, providerStubs = [], + enableRpcFailover = () => { + // do nothing, + }, + disableRpcFailover = () => { + // do nothing, + }, }: { configuration: NetworkClientConfiguration; providerStubs?: FakeProviderStub[]; + enableRpcFailover?: () => void; + disableRpcFailover?: () => void; }): NetworkClient { const provider = new FakeProvider({ stubs: providerStubs }); return { @@ -117,6 +130,8 @@ function buildFakeNetworkClient({ destroy: () => { // do nothing }, + enableRpcFailover, + disableRpcFailover, }; } diff --git a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts index 3bd7ed88736..576896ace64 100644 --- a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts +++ b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts @@ -410,9 +410,7 @@ export function testsForRpcMethodsThatCheckForBlockHashInResponse( async ({ makeRpcCall }) => makeRpcCall(request), ); - await expect(promiseForResult).rejects.toThrow( - `Non-200 status code: '${httpStatus}'`, - ); + await expect(promiseForResult).rejects.toThrow(errorMessage); }); }); diff --git a/packages/network-controller/tests/provider-api-tests/block-param.ts b/packages/network-controller/tests/provider-api-tests/block-param.ts index 038bdaa1ead..28ecc9e8fe0 100644 --- a/packages/network-controller/tests/provider-api-tests/block-param.ts +++ b/packages/network-controller/tests/provider-api-tests/block-param.ts @@ -753,7 +753,6 @@ export function testsForRpcMethodSupportingBlockParam( // The block-ref middleware will make the request as specified // except that the block param is replaced with the latest block // number. - comms.mockRpcCall({ request: buildRequestWithReplacedBlockParam( request, diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 9b3e1fad208..02965bc4dac 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -313,6 +313,7 @@ export type MockOptions = { getRpcServiceOptions?: NetworkControllerOptions['getRpcServiceOptions']; expectedHeaders?: Record; messenger?: RootMessenger; + isRpcFailoverEnabled?: boolean; }; export type MockCommunications = { @@ -473,6 +474,8 @@ export async function waitForPromiseToBeFulfilledAfterRunningAllTimers( * that `providerType` is "custom" (default: "ETH"). * @param options.getRpcServiceOptions - RPC service options factory. * @param options.messenger - The root messenger to use in tests. + * @param options.isRpcFailoverEnabled - Whether or not the RPC failover + * functionality is enabled. * @param fn - A function which will be called with an object that allows * interaction with the network client. * @returns The return value of the given function. @@ -487,6 +490,7 @@ export async function withNetworkClient( customTicker = 'ETH', getRpcServiceOptions = () => ({ fetch, btoa }), messenger = buildRootMessenger(), + isRpcFailoverEnabled = false, }: MockOptions, // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -538,6 +542,7 @@ export async function withNetworkClient( configuration: networkClientConfiguration, getRpcServiceOptions, messenger: networkControllerMessenger, + isRpcFailoverEnabled, }); /* eslint-disable-next-line n/no-process-env */ process.env.IN_TEST = inTest; diff --git a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts index 199d68619cf..7254c5a9670 100644 --- a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts +++ b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts @@ -40,7 +40,7 @@ export function testsForRpcFailoverBehavior({ ? maxConsecutiveFailures / (maxRetries + 1) : maxConsecutiveFailures; - describe('assuming RPC failover functionality is enabled', () => { + describe('if RPC failover functionality is enabled', () => { it(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( @@ -83,6 +83,7 @@ export function testsForRpcFailoverBehavior({ const result = await withNetworkClient( { providerType, + isRpcFailoverEnabled: true, failoverRpcUrls: ['https://failover.endpoint'], messenger, getRpcServiceOptions: () => ({ @@ -168,6 +169,7 @@ export function testsForRpcFailoverBehavior({ await withNetworkClient( { providerType, + isRpcFailoverEnabled: true, failoverRpcUrls: [failoverEndpointUrl], messenger, getRpcServiceOptions: () => ({ @@ -263,6 +265,7 @@ export function testsForRpcFailoverBehavior({ await withNetworkClient( { providerType, + isRpcFailoverEnabled: true, failoverRpcUrls: [failoverEndpointUrl], messenger, getRpcServiceOptions: () => ({ @@ -367,6 +370,7 @@ export function testsForRpcFailoverBehavior({ const result = await withNetworkClient( { providerType, + isRpcFailoverEnabled: true, failoverRpcUrls: ['https://failover.endpoint'], messenger, getRpcServiceOptions: (rpcEndpointUrl) => { @@ -427,4 +431,74 @@ export function testsForRpcFailoverBehavior({ ); }); }); + + describe('if RPC failover functionality is not enabled', () => { + it(`throws even after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { + await withMockedCommunications({ providerType }, async (comms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // block-cache middleware will request the latest block number + // through the block tracker to determine the cache key. Later, + // the block-ref middleware will request the latest block number + // again to resolve the value of "latest", but the block number is + // cached once made, so we only need to mock the request once. + comms.mockNextBlockTrackerRequest({ blockNumber }); + // The block-ref middleware will make the request as specified + // except that the block param is replaced with the latest block + // number. + comms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + + const messenger = buildRootMessenger(); + + await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: false, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, rpcUrl }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + const promiseForResult = makeRpcCall(request); + + await expect(promiseForResult).rejects.toThrow( + getExpectedError(rpcUrl), + ); + }, + ); + }); + }); + }); } diff --git a/packages/network-controller/tests/provider-api-tests/shared-tests.ts b/packages/network-controller/tests/provider-api-tests/shared-tests.ts index d49b05c2c7e..e98b55e1f48 100644 --- a/packages/network-controller/tests/provider-api-tests/shared-tests.ts +++ b/packages/network-controller/tests/provider-api-tests/shared-tests.ts @@ -131,7 +131,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe('methods that have a param to specify the block', () => { + describe.only('methods that have a param to specify the block', () => { const supportingBlockParam = [ { name: 'eth_call', diff --git a/packages/network-controller/tsconfig.build.json b/packages/network-controller/tsconfig.build.json index c054df5ef38..31f78014177 100644 --- a/packages/network-controller/tsconfig.build.json +++ b/packages/network-controller/tsconfig.build.json @@ -9,7 +9,8 @@ { "path": "../base-controller/tsconfig.build.json" }, { "path": "../controller-utils/tsconfig.build.json" }, { "path": "../eth-json-rpc-provider/tsconfig.build.json" }, - { "path": "../json-rpc-engine/tsconfig.build.json" } + { "path": "../json-rpc-engine/tsconfig.build.json" }, + { "path": "../remote-feature-flag-controller/tsconfig.build.json" } ], "include": ["../../types", "./src"] } diff --git a/packages/network-controller/tsconfig.json b/packages/network-controller/tsconfig.json index c6a988886f9..8362d23ec4f 100644 --- a/packages/network-controller/tsconfig.json +++ b/packages/network-controller/tsconfig.json @@ -16,6 +16,9 @@ }, { "path": "../json-rpc-engine" + }, + { + "path": "../remote-feature-flag-controller" } ], "include": ["../../types", "../../tests", "./src", "./tests"] diff --git a/packages/remote-feature-flag-controller/src/index.ts b/packages/remote-feature-flag-controller/src/index.ts index 2c5e2cd8025..c43f2dce998 100644 --- a/packages/remote-feature-flag-controller/src/index.ts +++ b/packages/remote-feature-flag-controller/src/index.ts @@ -1,5 +1,8 @@ export { RemoteFeatureFlagController } from './remote-feature-flag-controller'; -export type { RemoteFeatureFlagControllerMessenger } from './remote-feature-flag-controller'; +export type { + RemoteFeatureFlagControllerMessenger, + RemoteFeatureFlagControllerStateChangeEvent, +} from './remote-feature-flag-controller'; export { ClientType, DistributionType, diff --git a/yarn.lock b/yarn.lock index 76ce1189795..a91abd5cd24 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3818,6 +3818,7 @@ __metadata: "@metamask/eth-json-rpc-provider": "npm:^4.1.8" "@metamask/eth-query": "npm:^4.0.0" "@metamask/json-rpc-engine": "npm:^10.0.3" + "@metamask/remote-feature-flag-controller": "npm:^1.6.0" "@metamask/rpc-errors": "npm:^7.0.2" "@metamask/swappable-obj-proxy": "npm:^2.3.0" "@metamask/utils": "npm:^11.2.0" @@ -3843,6 +3844,8 @@ __metadata: typescript: "npm:~5.2.2" uri-js: "npm:^4.4.1" uuid: "npm:^8.3.2" + peerDependencies: + "@metamask/remote-feature-flag-controller": ^1.5.0 languageName: unknown linkType: soft From 70a15b9cb8c305d8bd073ff8ae741af48dd14dc7 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 13:15:32 -0600 Subject: [PATCH 03/17] wip --- .../src/create-network-client.ts | 18 +- .../src/rpc-service/rpc-service.ts | 12 + .../tests/create-network-client.test.ts | 4 + .../tests/provider-api-tests/helpers.ts | 17 +- .../provider-api-tests/no-block-param.ts | 10 +- .../tests/provider-api-tests/rpc-failover.ts | 699 +++++++++--------- .../tests/provider-api-tests/shared-tests.ts | 15 +- 7 files changed, 416 insertions(+), 359 deletions(-) diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index 23feb52b404..d995f42b01a 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -124,22 +124,24 @@ export function createNetworkClient({ }); const updateRpcServices = () => { - rpcServiceChain.updateServices( - determineAvailableEndpointUrls(isRpcFailoverEnabled).map( - (endpointUrl) => ({ - ...getRpcServiceOptions(endpointUrl), - endpointUrl, - }), - ), - ); + const rpcServiceConfigurations = determineAvailableEndpointUrls( + isRpcFailoverEnabled, + ).map((endpointUrl) => ({ + ...getRpcServiceOptions(endpointUrl), + endpointUrl, + })); + console.log('Rebuilding services', rpcServiceConfigurations); + rpcServiceChain.updateServices(rpcServiceConfigurations); }; const enableRpcFailover = () => { if (isRpcFailoverEnabled) { + console.log('enableRpcFailover: Already enabled?!'); return; } isRpcFailoverEnabled = true; + console.log('enableRpcFailover: Updating RPC services...'); updateRpcServices(); }; diff --git a/packages/network-controller/src/rpc-service/rpc-service.ts b/packages/network-controller/src/rpc-service/rpc-service.ts index e2766fd2a08..57e103b4a4b 100644 --- a/packages/network-controller/src/rpc-service/rpc-service.ts +++ b/packages/network-controller/src/rpc-service/rpc-service.ts @@ -384,10 +384,15 @@ export class RpcService implements AbstractRpcService { completeFetchOptions, ); } catch (error) { + console.log('Got error', error); if ( this.#policy.circuitBreakerPolicy.state === CircuitState.Open && this.#failoverService !== undefined ) { + console.log( + 'Failing over to', + this.#failoverService.endpointUrl.toString(), + ); return await this.#failoverService.request( jsonRpcRequest, completeFetchOptions, @@ -481,7 +486,14 @@ export class RpcService implements AbstractRpcService { fetchOptions: FetchOptions, ): Promise | JsonRpcResponse> { return await this.#policy.execute(async () => { + console.log( + 'Fetching', + this.endpointUrl.toString(), + 'with', + fetchOptions, + ); const response = await this.#fetch(this.endpointUrl, fetchOptions); + console.log('Got response from', this.endpointUrl.toString()); if (response.status === 405) { throw rpcErrors.methodNotFound(); diff --git a/packages/network-controller/tests/create-network-client.test.ts b/packages/network-controller/tests/create-network-client.test.ts index 6e425f4a3d0..b12be2e0e8a 100644 --- a/packages/network-controller/tests/create-network-client.test.ts +++ b/packages/network-controller/tests/create-network-client.test.ts @@ -2,6 +2,10 @@ import { NetworkClientType } from '../src/types'; import { testsForProviderType } from './provider-api-tests/shared-tests'; for (const clientType of Object.values(NetworkClientType)) { + if (clientType !== 'custom') { + continue; + } + describe(`createNetworkClient - ${clientType}`, () => { testsForProviderType(clientType); }); diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 02965bc4dac..3ad9e611557 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -168,7 +168,7 @@ function mockRpcCall({ ? `/v3/${MOCK_INFURA_PROJECT_ID}` : '/'; - debug('Mocking request:', { + debug('Nock Mocking Request:', { url, method, params, @@ -201,6 +201,7 @@ function mockRpcCall({ // eslint-disable-next-line @typescript-eslint/no-explicit-any return nockRequest.reply(httpStatus, (_, requestBody: any) => { if (typeof completeResponse === 'string') { + debug('Nock Returning Response', completeResponse); return completeResponse; } @@ -211,11 +212,12 @@ function mockRpcCall({ completeResponse.id = response.id; } } - debug('Nock returning Response', completeResponse); + debug('Nock Returning Response', completeResponse); return completeResponse; }); } - return nockRequest; + + throw new Error('No reply specified'); } type MockBlockTrackerRequestOptions = { @@ -391,6 +393,8 @@ type MockNetworkClient = { // eslint-disable-next-line @typescript-eslint/no-explicit-any blockTracker: any; provider: SafeEventEmitterProvider; + enableRpcFailover: () => void; + disableRpcFailover: () => void; clock: sinon.SinonFakeTimers; // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -547,13 +551,14 @@ export async function withNetworkClient( /* eslint-disable-next-line n/no-process-env */ process.env.IN_TEST = inTest; - const { provider, blockTracker } = networkClient; + const { provider, blockTracker, enableRpcFailover, disableRpcFailover } = + networkClient; const ethQuery = new EthQuery(provider); const curriedMakeRpcCall = (request: MockRequest) => makeRpcCall(ethQuery, request); const makeRpcCallsInSeries = async (requests: MockRequest[]) => { - const responses = []; + const responses: unknown[] = []; for (const request of requests) { responses.push(await curriedMakeRpcCall(request)); } @@ -563,6 +568,8 @@ export async function withNetworkClient( const client = { blockTracker, provider, + enableRpcFailover, + disableRpcFailover, clock, makeRpcCall: curriedMakeRpcCall, makeRpcCallsInSeries, diff --git a/packages/network-controller/tests/provider-api-tests/no-block-param.ts b/packages/network-controller/tests/provider-api-tests/no-block-param.ts index ef8dd12d54e..443b3bb36f9 100644 --- a/packages/network-controller/tests/provider-api-tests/no-block-param.ts +++ b/packages/network-controller/tests/provider-api-tests/no-block-param.ts @@ -292,7 +292,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - describe.each([ + describe.skip.each([ [405, 'The method does not exist / is not available.'], [429, 'Request is being rate limited.'], ])( @@ -343,7 +343,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe('if the RPC endpoint returns a response that is not 405, 429, 503, or 504', () => { + describe.skip('if the RPC endpoint returns a response that is not 405, 429, 503, or 504', () => { const httpStatus = 500; const errorMessage = `Non-200 status code: '${httpStatus}'`; @@ -391,7 +391,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - describe.each([503, 504])( + describe.skip.each([503, 504])( 'if the RPC endpoint returns a %d response', (httpStatus) => { const errorMessage = 'Gateway timeout'; @@ -487,7 +487,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe.each(['ETIMEDOUT', 'ECONNRESET'])( + describe.skip.each(['ETIMEDOUT', 'ECONNRESET'])( 'if a %s error is thrown while making the request', (errorCode) => { const error = new Error(errorCode); @@ -579,7 +579,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe('if the RPC endpoint responds with invalid JSON', () => { + describe.skip('if the RPC endpoint responds with invalid JSON', () => { const errorMessage = 'not valid JSON'; it('retries the request up to 5 times until it responds with valid JSON', async () => { diff --git a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts index 7254c5a9670..88acb780514 100644 --- a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts +++ b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts @@ -40,170 +40,198 @@ export function testsForRpcFailoverBehavior({ ? maxConsecutiveFailures / (maxRetries + 1) : maxConsecutiveFailures; - describe('if RPC failover functionality is enabled', () => { - it(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { - await withMockedCommunications({ providerType }, async (primaryComms) => { + describe.each([ + //{ + //extraDescription: 'at initialization', + //initialIsRpcFailoverEnabled: true, + //shouldEnableRpcFailoverAfterInitialization: false, + //}, + { + extraDescription: 'after initialization', + initialIsRpcFailoverEnabled: false, + shouldEnableRpcFailoverAfterInitialization: true, + }, + ])( + 'if RPC failover functionality is enabled $extraDescription', + ({ + initialIsRpcFailoverEnabled, + shouldEnableRpcFailoverAfterInitialization, + }) => { + it.only(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', + { providerType }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', }, - }); + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); - const messenger = buildRootMessenger(); + const messenger = buildRootMessenger(); - const result = await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), + const result = await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: initialIsRpcFailoverEnabled, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), }, - }), - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); + async ({ enableRpcFailover, makeRpcCall, clock }) => { + // eslint-disable-next-line jest/no-conditional-in-test + if (shouldEnableRpcFailoverAfterInitialization) { + enableRpcFailover(); + } + + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); }, ); - for (let i = 0; i < numRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); + expect(result).toBe('ok'); }, ); - - expect(result).toBe('ok'); }, ); }); - }); - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; - await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( + { providerType }, + async (primaryComms) => { + await withMockedCommunications( { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, }, - async ({ makeRpcCall, clock, chainId, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', }, - ); + }); - for (let i = 0; i < numRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); - expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( + await withNetworkClient( { - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl, - error: getExpectedError(rpcUrl), + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, chainId, rpcUrl }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + await makeRpcCall(request); + + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenCalledWith({ + chainId, + endpointUrl: rpcUrl, + failoverEndpointUrl, + error: getExpectedError(rpcUrl), + }); }, ); }, @@ -211,226 +239,229 @@ export function testsForRpcFailoverBehavior({ }, ); }); - }); - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; - await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, - }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); - messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( + { providerType }, + async (primaryComms) => { + await withMockedCommunications( { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, }, - async ({ makeRpcCall, clock, chainId }) => { + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // Block tracker requests on the primary will fail over + failoverComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, ); - for (let i = 0; i < maxConsecutiveFailures - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < maxConsecutiveFailures; i++) { - await ignoreRejection(makeRpcCall(request)); - } + await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, chainId }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); + + for (let i = 0; i < maxConsecutiveFailures - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + for (let i = 0; i < maxConsecutiveFailures; i++) { + await ignoreRejection(makeRpcCall(request)); + } - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: failoverEndpointUrl, - error: getExpectedError(failoverEndpointUrl), - }); + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenNthCalledWith(2, { + chainId, + endpointUrl: failoverEndpointUrl, + error: getExpectedError(failoverEndpointUrl), + }); + }, + ); }, ); }, ); }); - }); - it('allows RPC service options to be customized', async () => { - const customMaxConsecutiveFailures = 6; - const customMaxRetries = 2; - // This is okay. - // eslint-disable-next-line jest/no-conditional-in-test - const customNumRequestsToMake = isRetriableFailure - ? customMaxConsecutiveFailures / (customMaxRetries + 1) - : customMaxConsecutiveFailures; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, - }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Baz': 'Qux', - }, + it('allows RPC service options to be customized', async () => { + const customMaxConsecutiveFailures = 6; + const customMaxRetries = 2; + // This is okay. + // eslint-disable-next-line jest/no-conditional-in-test + const customNumRequestsToMake = isRetriableFailure + ? customMaxConsecutiveFailures / (customMaxRetries + 1) + : customMaxConsecutiveFailures; + + await withMockedCommunications( + { + providerType, + expectedHeaders: { + 'X-Foo': 'Bar', }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: customMaxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', + }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', + expectedHeaders: { + 'X-Baz': 'Qux', }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: customMaxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + + const result = await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: (rpcEndpointUrl) => { + const commonOptions = { fetch, btoa }; + // We need to return different results. + // eslint-disable-next-line jest/no-conditional-in-test + if (rpcEndpointUrl === 'https://failover.endpoint') { + const headers: HeadersInit = { + 'X-Baz': 'Qux', + }; + return { + ...commonOptions, + fetchOptions: { + headers, + }, + }; + } const headers: HeadersInit = { - 'X-Baz': 'Qux', + 'X-Foo': 'Bar', }; return { ...commonOptions, fetchOptions: { headers, }, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + maxRetries: customMaxRetries, + maxConsecutiveFailures: customMaxConsecutiveFailures, + }, }; - } - const headers: HeadersInit = { - 'X-Foo': 'Bar', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: customMaxRetries, - maxConsecutiveFailures: customMaxConsecutiveFailures, - }, - }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); }, - ); + }, + async ({ makeRpcCall, clock }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); - for (let i = 0; i < customNumRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); - }, - ); + for (let i = 0; i < customNumRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); + }, + ); - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - }); + expect(result).toBe('ok'); + }, + ); + }, + ); + }); + }, + ); describe('if RPC failover functionality is not enabled', () => { it(`throws even after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { diff --git a/packages/network-controller/tests/provider-api-tests/shared-tests.ts b/packages/network-controller/tests/provider-api-tests/shared-tests.ts index e98b55e1f48..04e64037e82 100644 --- a/packages/network-controller/tests/provider-api-tests/shared-tests.ts +++ b/packages/network-controller/tests/provider-api-tests/shared-tests.ts @@ -32,7 +32,7 @@ export function testsForProviderType(providerType: ProviderType) { // Ethereum JSON-RPC spec: // Infura documentation: describe('methods included in the Ethereum JSON-RPC spec', () => { - describe('methods not handled by middleware', () => { + describe.skip('methods not handled by middleware', () => { const notHandledByMiddleware = [ { name: 'eth_newFilter', numberOfParameters: 1 }, { name: 'eth_getFilterChanges', numberOfParameters: 1 }, @@ -74,7 +74,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe('methods with block hashes in their result', () => { + describe.skip('methods with block hashes in their result', () => { const methodsWithBlockHashInResponse = [ { name: 'eth_getTransactionByHash', numberOfParameters: 1 }, { name: 'eth_getTransactionReceipt', numberOfParameters: 1 }, @@ -121,6 +121,7 @@ export function testsForProviderType(providerType: ProviderType) { ]; assumingNoBlockParam .concat(blockParamIgnored) + .slice(0, 1) .forEach(({ name, numberOfParameters }) => describe(`method name: ${name}`, () => { testsForRpcMethodAssumingNoBlockParam(name, { @@ -131,7 +132,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe.only('methods that have a param to specify the block', () => { + describe.skip('methods that have a param to specify the block', () => { const supportingBlockParam = [ { name: 'eth_call', @@ -173,7 +174,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe('other methods', () => { + describe.skip('other methods', () => { describe('eth_getTransactionByHash', () => { it("refreshes the block tracker's current block if it is less than the block number that comes back in the response", async () => { const method = 'eth_getTransactionByHash'; @@ -249,7 +250,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe('methods not included in the Ethereum JSON-RPC spec', () => { + describe.skip('methods not included in the Ethereum JSON-RPC spec', () => { describe('methods not handled by middleware', () => { const notHandledByMiddleware = [ { name: 'net_listening', numberOfParameters: 0 }, @@ -269,7 +270,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe('methods that assume there is no block param', () => { + describe.skip('methods that assume there is no block param', () => { const assumingNoBlockParam = [ { name: 'web3_clientVersion', numberOfParameters: 0 }, { name: 'eth_protocolVersion', numberOfParameters: 0 }, @@ -284,7 +285,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe('other methods', () => { + describe.skip('other methods', () => { describe('net_version', () => { const networkArgs = { providerType, From 5fb8436007bef2eafce0530cbf323674762f5ed8 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 13:55:47 -0600 Subject: [PATCH 04/17] wip --- .../src/NetworkController.ts | 52 +- ...create-auto-managed-network-client.test.ts | 244 +----- .../src/create-auto-managed-network-client.ts | 37 +- .../src/create-network-client.ts | 55 +- .../src/rpc-service/rpc-service-chain.test.ts | 114 --- .../src/rpc-service/rpc-service-chain.ts | 21 +- .../tests/NetworkController.test.ts | 17 +- packages/network-controller/tests/helpers.ts | 13 - .../tests/provider-api-tests/helpers.ts | 7 +- .../tests/provider-api-tests/rpc-failover.ts | 699 +++++++++--------- 10 files changed, 413 insertions(+), 846 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index f39dce51eff..a7fa744dceb 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -1246,28 +1246,62 @@ export class NetworkController extends BaseController< } enableRpcFailover() { - if (this.#isRpcFailoverEnabled) { + this.#updateRpcFailoverEnabled(true); + } + + disableRpcFailover() { + this.#updateRpcFailoverEnabled(true); + } + + #updateRpcFailoverEnabled(newIsRpcFailoverEnabled: boolean) { + if (this.#isRpcFailoverEnabled === newIsRpcFailoverEnabled) { return; } - const networkClientsById = this.getNetworkClientRegistry(); + const autoManagedNetworkClientRegistry = + this.#ensureAutoManagedNetworkClientRegistryPopulated(); - for (const networkClient of Object.values(networkClientsById)) { + const infuraAutoManagedNetworkClientRegistry = + autoManagedNetworkClientRegistry[NetworkClientType.Infura]; + for (const networkClientId of knownKeysOf( + infuraAutoManagedNetworkClientRegistry, + )) { + const networkClient = + infuraAutoManagedNetworkClientRegistry[networkClientId]; console.log('networkClient', networkClient.configuration); if ( networkClient.configuration.failoverRpcUrls && networkClient.configuration.failoverRpcUrls.length > 0 ) { - networkClient.enableRpcFailover(); + networkClient.destroy(); + infuraAutoManagedNetworkClientRegistry[networkClientId] = + newIsRpcFailoverEnabled + ? networkClient.withRpcFailoverEnabled() + : networkClient.withRpcFailoverDisabled(); } } - this.#isRpcFailoverEnabled = true; - } + const customAutoManagedNetworkClientRegistry = + autoManagedNetworkClientRegistry[NetworkClientType.Custom]; + for (const networkClientId of knownKeysOf( + customAutoManagedNetworkClientRegistry, + )) { + const networkClient = + customAutoManagedNetworkClientRegistry[networkClientId]; + console.log('networkClient', networkClient.configuration); + if ( + networkClient.configuration.failoverRpcUrls && + networkClient.configuration.failoverRpcUrls.length > 0 + ) { + networkClient.destroy(); + customAutoManagedNetworkClientRegistry[networkClientId] = + newIsRpcFailoverEnabled + ? networkClient.withRpcFailoverEnabled() + : networkClient.withRpcFailoverDisabled(); + } + } - disableRpcFailover() { - // Go through all RPC endpoints with failover RPC URLs defined and call - // .enableRpcFailover on the network clients + this.#isRpcFailoverEnabled = newIsRpcFailoverEnabled; } /** diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index b89f0b5d9db..85626d6905c 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -192,7 +192,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover before the network client is initialized', async () => { + it('allows for enabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -223,8 +223,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: false, - }); - autoManagedNetworkClient.enableRpcFailover(); + }).withRpcFailoverEnabled(); const { provider } = autoManagedNetworkClient; await provider.request({ @@ -241,66 +240,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover after the network client is initialized', async () => { - const mockNetworkClient = buildFakeNetworkClient({ - configuration: networkClientConfiguration, - providerStubs: [ - { - request: { - method: 'test_method', - params: [], - }, - response: { - result: 'test response', - }, - }, - { - request: { - method: 'test_method', - params: [], - }, - response: { - result: 'test response', - }, - }, - ], - }); - const enableRpcFailoverMock = jest.spyOn( - mockNetworkClient, - 'enableRpcFailover', - ); - jest - .spyOn(createNetworkClientModule, 'createNetworkClient') - .mockReturnValue(mockNetworkClient); - const autoManagedNetworkClient = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - btoa, - fetch, - }), - messenger: getNetworkControllerMessenger(), - isRpcFailoverEnabled: false, - }); - const { provider } = autoManagedNetworkClient; - - await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], - }); - autoManagedNetworkClient.enableRpcFailover(); - await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], - }); - - expect(enableRpcFailoverMock).toHaveBeenCalled(); - }); - - it('allows for disabling the RPC failover before the network client is initialized', async () => { + it('allows for disabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -331,8 +271,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: true, - }); - autoManagedNetworkClient.disableRpcFailover(); + }).withRpcFailoverDisabled(); const { provider } = autoManagedNetworkClient; await provider.request({ @@ -348,65 +287,6 @@ describe('createAutoManagedNetworkClient', () => { isRpcFailoverEnabled: false, }); }); - - it('allows for disabling the RPC failover after the network client is initialized', async () => { - const mockNetworkClient = buildFakeNetworkClient({ - configuration: networkClientConfiguration, - providerStubs: [ - { - request: { - method: 'test_method', - params: [], - }, - response: { - result: 'test response', - }, - }, - { - request: { - method: 'test_method', - params: [], - }, - response: { - result: 'test response', - }, - }, - ], - }); - const disableRpcFailoverMock = jest.spyOn( - mockNetworkClient, - 'disableRpcFailover', - ); - jest - .spyOn(createNetworkClientModule, 'createNetworkClient') - .mockReturnValue(mockNetworkClient); - const autoManagedNetworkClient = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - btoa, - fetch, - }), - messenger: getNetworkControllerMessenger(), - isRpcFailoverEnabled: true, - }); - const { provider } = autoManagedNetworkClient; - - await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], - }); - autoManagedNetworkClient.disableRpcFailover(); - await provider.request({ - id: 1, - jsonrpc: '2.0', - method: 'test_method', - params: [], - }); - - expect(disableRpcFailoverMock).toHaveBeenCalled(); - }); }); it('returns a block tracker proxy that has the same interface as a block tracker', () => { @@ -559,7 +439,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover before the network client is initialized', async () => { + it('allows for enabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -589,8 +469,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: false, - }); - autoManagedNetworkClient.enableRpcFailover(); + }).withRpcFailoverEnabled(); const { blockTracker } = autoManagedNetworkClient; await new Promise((resolve) => { @@ -604,60 +483,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover after the network client is initialized', async () => { - const mockNetworkClient = buildFakeNetworkClient({ - configuration: networkClientConfiguration, - providerStubs: [ - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x1', - }, - }, - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x2', - }, - }, - ], - }); - const enableRpcFailoverMock = jest.spyOn( - mockNetworkClient, - 'enableRpcFailover', - ); - jest - .spyOn(createNetworkClientModule, 'createNetworkClient') - .mockReturnValue(mockNetworkClient); - const autoManagedNetworkClient = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - btoa, - fetch, - }), - messenger: getNetworkControllerMessenger(), - isRpcFailoverEnabled: false, - }); - const { blockTracker } = autoManagedNetworkClient; - - await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - autoManagedNetworkClient.enableRpcFailover(); - await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - - expect(enableRpcFailoverMock).toHaveBeenCalled(); - }); - - it('allows for disabling the RPC failover before the network client is initialized', async () => { + it('allows for disabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -687,8 +513,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: true, - }); - autoManagedNetworkClient.disableRpcFailover(); + }).withRpcFailoverDisabled(); const { blockTracker } = autoManagedNetworkClient; await new Promise((resolve) => { @@ -701,59 +526,6 @@ describe('createAutoManagedNetworkClient', () => { isRpcFailoverEnabled: false, }); }); - - it('allows for disabling the RPC failover after the network client is initialized', async () => { - const mockNetworkClient = buildFakeNetworkClient({ - configuration: networkClientConfiguration, - providerStubs: [ - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x1', - }, - }, - { - request: { - method: 'eth_blockNumber', - params: [], - }, - response: { - result: '0x2', - }, - }, - ], - }); - const disableRpcFailoverMock = jest.spyOn( - mockNetworkClient, - 'disableRpcFailover', - ); - jest - .spyOn(createNetworkClientModule, 'createNetworkClient') - .mockReturnValue(mockNetworkClient); - const autoManagedNetworkClient = createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions: () => ({ - btoa, - fetch, - }), - messenger: getNetworkControllerMessenger(), - isRpcFailoverEnabled: true, - }); - const { blockTracker } = autoManagedNetworkClient; - - await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - autoManagedNetworkClient.disableRpcFailover(); - await new Promise((resolve) => { - blockTracker.once('latest', resolve); - }); - - expect(disableRpcFailoverMock).toHaveBeenCalled(); - }); }); }); diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index d027bf04030..cbc1185ba88 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -40,8 +40,8 @@ export type AutoManagedNetworkClient< provider: ProxyWithAccessibleTarget; blockTracker: ProxyWithAccessibleTarget; destroy: () => void; - enableRpcFailover: () => void; - disableRpcFailover: () => void; + withRpcFailoverEnabled: () => AutoManagedNetworkClient; + withRpcFailoverDisabled: () => AutoManagedNetworkClient; }; /** @@ -80,7 +80,7 @@ export function createAutoManagedNetworkClient< networkClientConfiguration, getRpcServiceOptions, messenger, - isRpcFailoverEnabled: initialRpcFailoverEnabled, + isRpcFailoverEnabled, }: { networkClientConfiguration: Configuration; getRpcServiceOptions: ( @@ -89,7 +89,6 @@ export function createAutoManagedNetworkClient< messenger: NetworkControllerMessenger; isRpcFailoverEnabled: boolean; }): AutoManagedNetworkClient { - let isRpcFailoverEnabled = initialRpcFailoverEnabled; let networkClient: NetworkClient | undefined; const ensureNetworkClientCreated = (): NetworkClient => { @@ -204,20 +203,22 @@ export function createAutoManagedNetworkClient< networkClient?.destroy(); }; - const enableRpcFailover = () => { - if (networkClient) { - networkClient.enableRpcFailover(); - } else { - isRpcFailoverEnabled = true; - } + const withRpcFailoverEnabled = () => { + return createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); }; - const disableRpcFailover = () => { - if (networkClient) { - networkClient.disableRpcFailover(); - } else { - isRpcFailoverEnabled = false; - } + const withRpcFailoverDisabled = () => { + return createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); }; return { @@ -225,7 +226,7 @@ export function createAutoManagedNetworkClient< provider: providerProxy, blockTracker: blockTrackerProxy, destroy, - enableRpcFailover, - disableRpcFailover, + withRpcFailoverEnabled, + withRpcFailoverDisabled, }; } diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index d995f42b01a..cc0aa7ee274 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -46,8 +46,6 @@ export type NetworkClient = { provider: Provider; blockTracker: BlockTracker; destroy: () => void; - enableRpcFailover: () => void; - disableRpcFailover: () => void; }; /** @@ -68,7 +66,7 @@ export function createNetworkClient({ configuration, getRpcServiceOptions, messenger, - isRpcFailoverEnabled: initialRpcFailoverEnabled, + isRpcFailoverEnabled, }: { configuration: NetworkClientConfiguration; getRpcServiceOptions: ( @@ -77,18 +75,13 @@ export function createNetworkClient({ messenger: NetworkControllerMessenger; isRpcFailoverEnabled: boolean; }): NetworkClient { - let isRpcFailoverEnabled = initialRpcFailoverEnabled; const primaryEndpointUrl = configuration.type === NetworkClientType.Infura ? `https://${configuration.network}.infura.io/v3/${configuration.infuraProjectId}` : configuration.rpcUrl; - const determineAvailableEndpointUrls = (givenRpcFailoverEnabled: boolean) => { - return givenRpcFailoverEnabled - ? [primaryEndpointUrl, ...(configuration.failoverRpcUrls ?? [])] - : [primaryEndpointUrl]; - }; - const availableEndpointUrls = - determineAvailableEndpointUrls(isRpcFailoverEnabled); + const availableEndpointUrls = isRpcFailoverEnabled + ? [primaryEndpointUrl, ...(configuration.failoverRpcUrls ?? [])] + : [primaryEndpointUrl]; const rpcServiceChain = new RpcServiceChain( availableEndpointUrls.map((endpointUrl) => ({ ...getRpcServiceOptions(endpointUrl), @@ -123,37 +116,6 @@ export function createNetworkClient({ }); }); - const updateRpcServices = () => { - const rpcServiceConfigurations = determineAvailableEndpointUrls( - isRpcFailoverEnabled, - ).map((endpointUrl) => ({ - ...getRpcServiceOptions(endpointUrl), - endpointUrl, - })); - console.log('Rebuilding services', rpcServiceConfigurations); - rpcServiceChain.updateServices(rpcServiceConfigurations); - }; - - const enableRpcFailover = () => { - if (isRpcFailoverEnabled) { - console.log('enableRpcFailover: Already enabled?!'); - return; - } - - isRpcFailoverEnabled = true; - console.log('enableRpcFailover: Updating RPC services...'); - updateRpcServices(); - }; - - const disableRpcFailover = () => { - if (!isRpcFailoverEnabled) { - return; - } - - isRpcFailoverEnabled = false; - updateRpcServices(); - }; - const rpcApiMiddleware = configuration.type === NetworkClientType.Infura ? createInfuraMiddleware({ @@ -201,14 +163,7 @@ export function createNetworkClient({ blockTracker.destroy(); }; - return { - configuration, - provider, - blockTracker, - destroy, - enableRpcFailover, - disableRpcFailover, - }; + return { configuration, provider, blockTracker, destroy }; } /** diff --git a/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts b/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts index 301c9eb34d5..c4edfd921a7 100644 --- a/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts +++ b/packages/network-controller/src/rpc-service/rpc-service-chain.test.ts @@ -15,120 +15,6 @@ describe('RpcServiceChain', () => { clock.restore(); }); - describe('updateServices', () => { - it('replaces the underlying RPC services with a new set constructed from the given configuration objects', async () => { - nock('https://first.chain') - .post( - '/', - { - id: 1, - jsonrpc: '2.0', - method: 'eth_chainId', - params: [], - }, - { - reqheaders: {}, - }, - ) - .times(15) - .reply(503); - nock('https://second.chain') - .post('/', { - id: 1, - jsonrpc: '2.0', - method: 'eth_chainId', - params: [], - }) - .times(15) - .reply(503); - nock('https://third.chain') - .post('/', { - id: 1, - jsonrpc: '2.0', - method: 'eth_chainId', - params: [], - }) - .reply(200, { - id: 1, - jsonrpc: '2.0', - result: 'ok', - }); - - const rpcServiceChain = new RpcServiceChain([ - { - fetch, - btoa, - endpointUrl: 'https://some.other.chain', - }, - ]); - rpcServiceChain.updateServices([ - { - fetch, - btoa, - endpointUrl: 'https://first.chain', - }, - { - fetch, - btoa, - endpointUrl: 'https://second.chain', - fetchOptions: { - headers: { - 'X-Foo': 'Bar', - }, - }, - }, - { - fetch, - btoa, - endpointUrl: 'https://third.chain', - }, - ]); - rpcServiceChain.onRetry(() => { - // We don't need to await this promise; adding it to the promise - // queue is enough to continue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.nextAsync(); - }); - - const jsonRpcRequest = { - id: 1, - jsonrpc: '2.0' as const, - method: 'eth_chainId', - params: [], - }; - // Retry the first endpoint until max retries is hit. - await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( - 'Gateway timeout', - ); - // Retry the first endpoint again, until max retries is hit. - await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( - 'Gateway timeout', - ); - // Retry the first endpoint for a third time, until max retries is hit. - // The circuit will break on the last time, and the second endpoint will - // be retried, until max retries is hit. - await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( - 'Gateway timeout', - ); - // Try the first endpoint, see that the circuit is broken, and retry the - // second endpoint, until max retries is hit. - await expect(rpcServiceChain.request(jsonRpcRequest)).rejects.toThrow( - 'Gateway timeout', - ); - // Try the first endpoint, see that the circuit is broken, and retry the - // second endpoint, until max retries is hit. - // The circuit will break on the last time, and the third endpoint will - // be hit. This is finally a success. - const response = await rpcServiceChain.request(jsonRpcRequest); - - expect(response).toStrictEqual({ - id: 1, - jsonrpc: '2.0', - result: 'ok', - }); - }); - }); - describe('onRetry', () => { it('returns a listener which can be disposed', () => { const rpcServiceChain = new RpcServiceChain([ diff --git a/packages/network-controller/src/rpc-service/rpc-service-chain.ts b/packages/network-controller/src/rpc-service/rpc-service-chain.ts index 73595728cea..65921f27695 100644 --- a/packages/network-controller/src/rpc-service/rpc-service-chain.ts +++ b/packages/network-controller/src/rpc-service/rpc-service-chain.ts @@ -17,7 +17,7 @@ import type { FetchOptions } from './shared'; * failovers. */ export class RpcServiceChain implements RpcServiceRequestable { - #services: RpcService[] = []; + readonly #services: RpcService[]; /** * Constructs a new RpcServiceChain object. @@ -28,25 +28,6 @@ export class RpcServiceChain implements RpcServiceRequestable { */ constructor( rpcServiceConfigurations: Omit[], - ) { - this.updateServices(rpcServiceConfigurations); - } - - /** - * Replaces the underlying RPC services with a new set constructed from the - * given configuration objects. - * - * This is useful when toggling the RPC failover feature without needing to - * reconstruct entire network clients or RPC middleware stacks. (This is - * possible because the fetch and Infura middleware take this whole - * RpcServiceChain object, not individual RpcService instances.) - * - * @param rpcServiceConfigurations - The options for the RPC services - * that you want to construct. Each object in this array is the same as - * {@link RpcServiceOptions}. - */ - updateServices( - rpcServiceConfigurations: Omit[], ) { this.#services = this.#buildRpcServiceChain(rpcServiceConfigurations); } diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index 261cfc62bf2..15bde6ae34a 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -699,7 +699,7 @@ describe('NetworkController', () => { .mockImplementation((...args) => { const autoManagedNetworkClient = originalCreateAutoManagedNetworkClient(...args); - jest.spyOn(autoManagedNetworkClient, 'enableRpcFailover'); + jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverEnabled'); autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); @@ -752,7 +752,7 @@ describe('NetworkController', () => { expect(autoManagedNetworkClients).toHaveLength(3); for (const autoManagedNetworkClient of autoManagedNetworkClients) { expect( - autoManagedNetworkClient.enableRpcFailover, + autoManagedNetworkClient.withRpcFailoverEnabled, ).toHaveBeenCalled(); } }, @@ -14974,17 +14974,6 @@ async function withController( */ function buildFakeClient( provider: Provider = buildFakeProvider(), - { - enableRpcFailover = () => { - // do nothing - }, - disableRpcFailover = () => { - // do nothing - }, - }: { - enableRpcFailover?: () => void; - disableRpcFailover?: () => void; - } = {}, ): NetworkClient { return { configuration: { @@ -14999,8 +14988,6 @@ function buildFakeClient( destroy: () => { // do nothing }, - enableRpcFailover, - disableRpcFailover, }; } diff --git a/packages/network-controller/tests/helpers.ts b/packages/network-controller/tests/helpers.ts index b5d5b4da3b4..09ffb1938f2 100644 --- a/packages/network-controller/tests/helpers.ts +++ b/packages/network-controller/tests/helpers.ts @@ -102,25 +102,14 @@ export function buildNetworkControllerMessenger( * @param args.configuration - The desired network client configuration. * @param args.providerStubs - Objects that allow for stubbing specific provider * requests. - * @param args.enableRpcFailover - Override for the `enableRpcFailover` method. - * @param args.disableRpcFailover - Override for the `disableRpcFailover` - * method. * @returns The fake network client. */ export function buildFakeNetworkClient({ configuration, providerStubs = [], - enableRpcFailover = () => { - // do nothing, - }, - disableRpcFailover = () => { - // do nothing, - }, }: { configuration: NetworkClientConfiguration; providerStubs?: FakeProviderStub[]; - enableRpcFailover?: () => void; - disableRpcFailover?: () => void; }): NetworkClient { const provider = new FakeProvider({ stubs: providerStubs }); return { @@ -130,8 +119,6 @@ export function buildFakeNetworkClient({ destroy: () => { // do nothing }, - enableRpcFailover, - disableRpcFailover, }; } diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 3ad9e611557..29cdea4b2a8 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -393,8 +393,6 @@ type MockNetworkClient = { // eslint-disable-next-line @typescript-eslint/no-explicit-any blockTracker: any; provider: SafeEventEmitterProvider; - enableRpcFailover: () => void; - disableRpcFailover: () => void; clock: sinon.SinonFakeTimers; // TODO: Replace `any` with type // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -551,8 +549,7 @@ export async function withNetworkClient( /* eslint-disable-next-line n/no-process-env */ process.env.IN_TEST = inTest; - const { provider, blockTracker, enableRpcFailover, disableRpcFailover } = - networkClient; + const { provider, blockTracker } = networkClient; const ethQuery = new EthQuery(provider); const curriedMakeRpcCall = (request: MockRequest) => @@ -568,8 +565,6 @@ export async function withNetworkClient( const client = { blockTracker, provider, - enableRpcFailover, - disableRpcFailover, clock, makeRpcCall: curriedMakeRpcCall, makeRpcCallsInSeries, diff --git a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts index 88acb780514..5fb8c532b33 100644 --- a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts +++ b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts @@ -40,198 +40,170 @@ export function testsForRpcFailoverBehavior({ ? maxConsecutiveFailures / (maxRetries + 1) : maxConsecutiveFailures; - describe.each([ - //{ - //extraDescription: 'at initialization', - //initialIsRpcFailoverEnabled: true, - //shouldEnableRpcFailoverAfterInitialization: false, - //}, - { - extraDescription: 'after initialization', - initialIsRpcFailoverEnabled: false, - shouldEnableRpcFailoverAfterInitialization: true, - }, - ])( - 'if RPC failover functionality is enabled $extraDescription', - ({ - initialIsRpcFailoverEnabled, - shouldEnableRpcFailoverAfterInitialization, - }) => { - it.only(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { + describe('if RPC failover functionality is enabled', () => { + it.only(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { + await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', - }, - }); + }); - const messenger = buildRootMessenger(); + const messenger = buildRootMessenger(); - const result = await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: initialIsRpcFailoverEnabled, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), + const result = await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), }, - async ({ enableRpcFailover, makeRpcCall, clock }) => { - // eslint-disable-next-line jest/no-conditional-in-test - if (shouldEnableRpcFailoverAfterInitialization) { - enableRpcFailover(); - } - - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < numRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - return await makeRpcCall(request); + }), + }, + async ({ makeRpcCall, clock }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); }, ); - expect(result).toBe('ok'); + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); }, ); + + expect(result).toBe('ok'); }, ); }); + }); - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover occurs', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', - }, - }); + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); + await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), + }, + async ({ makeRpcCall, clock, chainId, rpcUrl }) => { messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, ); - await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), - }, - async ({ makeRpcCall, clock, chainId, rpcUrl }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); + for (let i = 0; i < numRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + await makeRpcCall(request); - for (let i = 0; i < numRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - await makeRpcCall(request); - - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenCalledWith({ - chainId, - endpointUrl: rpcUrl, - failoverEndpointUrl, - error: getExpectedError(rpcUrl), - }); + expect(rpcEndpointUnavailableEventHandler).toHaveBeenCalledWith( + { + chainId, + endpointUrl: rpcUrl, + failoverEndpointUrl, + error: getExpectedError(rpcUrl), }, ); }, @@ -239,229 +211,226 @@ export function testsForRpcFailoverBehavior({ }, ); }); + }); - it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { - const failoverEndpointUrl = 'https://failover.endpoint/'; + it('publishes the NetworkController:rpcEndpointUnavailable event when the failover becomes unavailable', async () => { + const failoverEndpointUrl = 'https://failover.endpoint/'; + await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( - { providerType }, - async (primaryComms) => { - await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: failoverEndpointUrl, + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + times: maxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // Block tracker requests on the primary will fail over + failoverComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + + const messenger = buildRootMessenger(); + const rpcEndpointUnavailableEventHandler = jest.fn(); + messenger.subscribe( + 'NetworkController:rpcEndpointUnavailable', + rpcEndpointUnavailableEventHandler, + ); + + await withNetworkClient( { - providerType: 'custom', - customRpcUrl: failoverEndpointUrl, + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: [failoverEndpointUrl], + messenger, + getRpcServiceOptions: () => ({ + fetch, + btoa, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + }, + }), }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - times: maxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // Block tracker requests on the primary will fail over - failoverComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - - const messenger = buildRootMessenger(); - const rpcEndpointUnavailableEventHandler = jest.fn(); + async ({ makeRpcCall, clock, chainId }) => { messenger.subscribe( - 'NetworkController:rpcEndpointUnavailable', - rpcEndpointUnavailableEventHandler, - ); - - await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: [failoverEndpointUrl], - messenger, - getRpcServiceOptions: () => ({ - fetch, - btoa, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - }, - }), + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); }, - async ({ makeRpcCall, clock, chainId }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); + ); - for (let i = 0; i < maxConsecutiveFailures - 1; i++) { - await ignoreRejection(makeRpcCall(request)); - } - for (let i = 0; i < maxConsecutiveFailures; i++) { - await ignoreRejection(makeRpcCall(request)); - } + for (let i = 0; i < maxConsecutiveFailures - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + for (let i = 0; i < maxConsecutiveFailures; i++) { + await ignoreRejection(makeRpcCall(request)); + } - expect( - rpcEndpointUnavailableEventHandler, - ).toHaveBeenNthCalledWith(2, { - chainId, - endpointUrl: failoverEndpointUrl, - error: getExpectedError(failoverEndpointUrl), - }); - }, - ); + expect( + rpcEndpointUnavailableEventHandler, + ).toHaveBeenNthCalledWith(2, { + chainId, + endpointUrl: failoverEndpointUrl, + error: getExpectedError(failoverEndpointUrl), + }); }, ); }, ); }); + }); - it('allows RPC service options to be customized', async () => { - const customMaxConsecutiveFailures = 6; - const customMaxRetries = 2; - // This is okay. - // eslint-disable-next-line jest/no-conditional-in-test - const customNumRequestsToMake = isRetriableFailure - ? customMaxConsecutiveFailures / (customMaxRetries + 1) - : customMaxConsecutiveFailures; - - await withMockedCommunications( - { - providerType, - expectedHeaders: { - 'X-Foo': 'Bar', - }, + it('allows RPC service options to be customized', async () => { + const customMaxConsecutiveFailures = 6; + const customMaxRetries = 2; + // This is okay. + // eslint-disable-next-line jest/no-conditional-in-test + const customNumRequestsToMake = isRetriableFailure + ? customMaxConsecutiveFailures / (customMaxRetries + 1) + : customMaxConsecutiveFailures; + + await withMockedCommunications( + { + providerType, + expectedHeaders: { + 'X-Foo': 'Bar', }, - async (primaryComms) => { - await withMockedCommunications( - { - providerType: 'custom', - customRpcUrl: 'https://failover.endpoint', - expectedHeaders: { - 'X-Baz': 'Qux', - }, + }, + async (primaryComms) => { + await withMockedCommunications( + { + providerType: 'custom', + customRpcUrl: 'https://failover.endpoint', + expectedHeaders: { + 'X-Baz': 'Qux', }, - async (failoverComms) => { - const request = requestToCall; - const requestToMock = getRequestToMock(request, blockNumber); - const additionalMockRpcCallOptions = - // eslint-disable-next-line jest/no-conditional-in-test - failure instanceof Error || typeof failure === 'string' - ? { error: failure } - : { response: failure }; - - // The first time a block-cacheable request is made, the - // latest block number is retrieved through the block - // tracker first. - primaryComms.mockNextBlockTrackerRequest({ - blockNumber, - }); - primaryComms.mockRpcCall({ - request: requestToMock, - times: customMaxConsecutiveFailures, - ...additionalMockRpcCallOptions, - }); - // The block-ref middleware will make the request as - // specified except that the block param is replaced with - // the latest block number. - failoverComms.mockRpcCall({ - request: requestToMock, - response: { - result: 'ok', - }, - }); - - const messenger = buildRootMessenger(); - - const result = await withNetworkClient( - { - providerType, - isRpcFailoverEnabled: true, - failoverRpcUrls: ['https://failover.endpoint'], - messenger, - getRpcServiceOptions: (rpcEndpointUrl) => { - const commonOptions = { fetch, btoa }; - // We need to return different results. - // eslint-disable-next-line jest/no-conditional-in-test - if (rpcEndpointUrl === 'https://failover.endpoint') { - const headers: HeadersInit = { - 'X-Baz': 'Qux', - }; - return { - ...commonOptions, - fetchOptions: { - headers, - }, - }; - } + }, + async (failoverComms) => { + const request = requestToCall; + const requestToMock = getRequestToMock(request, blockNumber); + const additionalMockRpcCallOptions = + // eslint-disable-next-line jest/no-conditional-in-test + failure instanceof Error || typeof failure === 'string' + ? { error: failure } + : { response: failure }; + + // The first time a block-cacheable request is made, the + // latest block number is retrieved through the block + // tracker first. + primaryComms.mockNextBlockTrackerRequest({ + blockNumber, + }); + primaryComms.mockRpcCall({ + request: requestToMock, + times: customMaxConsecutiveFailures, + ...additionalMockRpcCallOptions, + }); + // The block-ref middleware will make the request as + // specified except that the block param is replaced with + // the latest block number. + failoverComms.mockRpcCall({ + request: requestToMock, + response: { + result: 'ok', + }, + }); + + const messenger = buildRootMessenger(); + + const result = await withNetworkClient( + { + providerType, + isRpcFailoverEnabled: true, + failoverRpcUrls: ['https://failover.endpoint'], + messenger, + getRpcServiceOptions: (rpcEndpointUrl) => { + const commonOptions = { fetch, btoa }; + // We need to return different results. + // eslint-disable-next-line jest/no-conditional-in-test + if (rpcEndpointUrl === 'https://failover.endpoint') { const headers: HeadersInit = { - 'X-Foo': 'Bar', + 'X-Baz': 'Qux', }; return { ...commonOptions, fetchOptions: { headers, }, - policyOptions: { - backoff: new ConstantBackoff(backoffDuration), - maxRetries: customMaxRetries, - maxConsecutiveFailures: customMaxConsecutiveFailures, - }, }; - }, - }, - async ({ makeRpcCall, clock }) => { - messenger.subscribe( - 'NetworkController:rpcEndpointRequestRetried', - () => { - // Ensure that we advance to the next RPC request - // retry, not the next block tracker request. - // We also don't need to await this, it just needs to - // be added to the promise queue. - // eslint-disable-next-line @typescript-eslint/no-floating-promises - clock.tickAsync(backoffDuration); - }, - ); - - for (let i = 0; i < customNumRequestsToMake - 1; i++) { - await ignoreRejection(makeRpcCall(request)); } - return await makeRpcCall(request); + const headers: HeadersInit = { + 'X-Foo': 'Bar', + }; + return { + ...commonOptions, + fetchOptions: { + headers, + }, + policyOptions: { + backoff: new ConstantBackoff(backoffDuration), + maxRetries: customMaxRetries, + maxConsecutiveFailures: customMaxConsecutiveFailures, + }, + }; }, - ); + }, + async ({ makeRpcCall, clock }) => { + messenger.subscribe( + 'NetworkController:rpcEndpointRequestRetried', + () => { + // Ensure that we advance to the next RPC request + // retry, not the next block tracker request. + // We also don't need to await this, it just needs to + // be added to the promise queue. + // eslint-disable-next-line @typescript-eslint/no-floating-promises + clock.tickAsync(backoffDuration); + }, + ); - expect(result).toBe('ok'); - }, - ); - }, - ); - }); - }, - ); + for (let i = 0; i < customNumRequestsToMake - 1; i++) { + await ignoreRejection(makeRpcCall(request)); + } + return await makeRpcCall(request); + }, + ); + + expect(result).toBe('ok'); + }, + ); + }, + ); + }); + }); describe('if RPC failover functionality is not enabled', () => { it(`throws even after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { From db8c195c8bd7bb7a38e70758d2fe46ade62ba4f2 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 14:14:17 -0600 Subject: [PATCH 05/17] done --- .../src/NetworkController.ts | 4 +- ...create-auto-managed-network-client.test.ts | 4 - .../src/rpc-service/rpc-service.ts | 12 - .../tests/NetworkController.test.ts | 274 ++++++++++++++---- .../tests/create-network-client.test.ts | 4 - .../provider-api-tests/no-block-param.ts | 10 +- .../tests/provider-api-tests/rpc-failover.ts | 2 +- .../tests/provider-api-tests/shared-tests.ts | 14 +- 8 files changed, 236 insertions(+), 88 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index a7fa744dceb..3448b05d79d 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -1250,7 +1250,7 @@ export class NetworkController extends BaseController< } disableRpcFailover() { - this.#updateRpcFailoverEnabled(true); + this.#updateRpcFailoverEnabled(false); } #updateRpcFailoverEnabled(newIsRpcFailoverEnabled: boolean) { @@ -1268,7 +1268,6 @@ export class NetworkController extends BaseController< )) { const networkClient = infuraAutoManagedNetworkClientRegistry[networkClientId]; - console.log('networkClient', networkClient.configuration); if ( networkClient.configuration.failoverRpcUrls && networkClient.configuration.failoverRpcUrls.length > 0 @@ -1288,7 +1287,6 @@ export class NetworkController extends BaseController< )) { const networkClient = customAutoManagedNetworkClientRegistry[networkClientId]; - console.log('networkClient', networkClient.configuration); if ( networkClient.configuration.failoverRpcUrls && networkClient.configuration.failoverRpcUrls.length > 0 diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index 85626d6905c..01f00cad5f8 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -13,10 +13,6 @@ import type { } from './types'; import { NetworkClientType } from './types'; import { mockNetwork } from '../../../tests/mock-network'; -import { - buildCustomNetworkClientConfiguration, - buildFakeNetworkClient, -} from '../tests/helpers'; describe('createAutoManagedNetworkClient', () => { const networkClientConfigurations: [ diff --git a/packages/network-controller/src/rpc-service/rpc-service.ts b/packages/network-controller/src/rpc-service/rpc-service.ts index 57e103b4a4b..e2766fd2a08 100644 --- a/packages/network-controller/src/rpc-service/rpc-service.ts +++ b/packages/network-controller/src/rpc-service/rpc-service.ts @@ -384,15 +384,10 @@ export class RpcService implements AbstractRpcService { completeFetchOptions, ); } catch (error) { - console.log('Got error', error); if ( this.#policy.circuitBreakerPolicy.state === CircuitState.Open && this.#failoverService !== undefined ) { - console.log( - 'Failing over to', - this.#failoverService.endpointUrl.toString(), - ); return await this.#failoverService.request( jsonRpcRequest, completeFetchOptions, @@ -486,14 +481,7 @@ export class RpcService implements AbstractRpcService { fetchOptions: FetchOptions, ): Promise | JsonRpcResponse> { return await this.#policy.execute(async () => { - console.log( - 'Fetching', - this.endpointUrl.toString(), - 'with', - fetchOptions, - ); const response = await this.#fetch(this.endpointUrl, fetchOptions); - console.log('Got response from', this.endpointUrl.toString()); if (response.status === 405) { throw rpcErrors.methodNotFound(); diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index 15bde6ae34a..b0dcc0400c6 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -642,14 +642,11 @@ describe('NetworkController', () => { }); describe('enableRpcFailover', () => { - describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { - it.todo('does nothing'); - }); - describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { - it.only('calls enableRpcFailover on only the network clients whose RPC endpoints have configured failover URLs', async () => { + it.only('calls withRpcFailoverEnabled on only the network clients whose RPC endpoints have configured failover URLs', async () => { await withController( { + isRpcFailoverEnabled: false, state: { selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', networkConfigurationsByChainId: { @@ -703,58 +700,231 @@ describe('NetworkController', () => { autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); - /* - const fakeAutoManagedNetworkClients = [ - { - enableRpcFailover: jest.fn(), - }, - { - enableRpcFailover: jest.fn(), - }, - { - enableRpcFailover: jest.fn(), - }, - ]; - createAutoManagedNetworkClientSpy.mockImplementation( - // @ts-expect-error We are intentionally returning partial - // AutoManagedCustomNetworkClients. - ({ networkClientConfiguration }) => { - if ( - networkClientConfiguration.type === NetworkClientType.Infura - ) { - return fakeAutoManagedNetworkClients[0]; - } - if ( - networkClientConfiguration.type === - NetworkClientType.Custom && - networkClientConfiguration.rpcUrl === 'https://test.network/1' - ) { - return fakeAutoManagedNetworkClients[1]; - } - if ( - networkClientConfiguration.type === - NetworkClientType.Custom && - networkClientConfiguration.rpcUrl === 'https://test.network/2' - ) { - return fakeAutoManagedNetworkClients[2]; - } - throw new Error( - `Unknown network client configuration ${JSON.stringify( - networkClientConfiguration, - )}`, - ); + + controller.enableRpcFailover(); + + expect(autoManagedNetworkClients).toHaveLength(3); + expect( + autoManagedNetworkClients[0].withRpcFailoverEnabled, + ).not.toHaveBeenCalled(); + expect( + autoManagedNetworkClients[1].withRpcFailoverEnabled, + ).toHaveBeenCalled(); + expect( + autoManagedNetworkClients[2].withRpcFailoverEnabled, + ).toHaveBeenCalled(); + }, + ); + }); + }); + + describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { + it.only('does not call createAutoManagedNetworkClient at all', async () => { + await withController( + { + isRpcFailoverEnabled: true, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), }, - ); - */ + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverEnabled'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); controller.enableRpcFailover(); + expect(autoManagedNetworkClients).toHaveLength(0); + }, + ); + }); + }); + }); + + describe('disableRpcFailover', () => { + describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { + it.only('calls withRpcFailoverDisabled on only the network clients whose RPC endpoints have configured failover URLs', async () => { + await withController( + { + isRpcFailoverEnabled: true, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverDisabled'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); + + controller.disableRpcFailover(); + expect(autoManagedNetworkClients).toHaveLength(3); - for (const autoManagedNetworkClient of autoManagedNetworkClients) { - expect( - autoManagedNetworkClient.withRpcFailoverEnabled, - ).toHaveBeenCalled(); - } + expect( + autoManagedNetworkClients[0].withRpcFailoverDisabled, + ).not.toHaveBeenCalled(); + expect( + autoManagedNetworkClients[1].withRpcFailoverDisabled, + ).toHaveBeenCalled(); + expect( + autoManagedNetworkClients[2].withRpcFailoverDisabled, + ).toHaveBeenCalled(); + }, + ); + }); + }); + + describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { + it.only('does not call createAutoManagedNetworkClient at all', async () => { + await withController( + { + isRpcFailoverEnabled: false, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverDisabled'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); + + controller.disableRpcFailover(); + + expect(autoManagedNetworkClients).toHaveLength(0); }, ); }); diff --git a/packages/network-controller/tests/create-network-client.test.ts b/packages/network-controller/tests/create-network-client.test.ts index b12be2e0e8a..6e425f4a3d0 100644 --- a/packages/network-controller/tests/create-network-client.test.ts +++ b/packages/network-controller/tests/create-network-client.test.ts @@ -2,10 +2,6 @@ import { NetworkClientType } from '../src/types'; import { testsForProviderType } from './provider-api-tests/shared-tests'; for (const clientType of Object.values(NetworkClientType)) { - if (clientType !== 'custom') { - continue; - } - describe(`createNetworkClient - ${clientType}`, () => { testsForProviderType(clientType); }); diff --git a/packages/network-controller/tests/provider-api-tests/no-block-param.ts b/packages/network-controller/tests/provider-api-tests/no-block-param.ts index 443b3bb36f9..ef8dd12d54e 100644 --- a/packages/network-controller/tests/provider-api-tests/no-block-param.ts +++ b/packages/network-controller/tests/provider-api-tests/no-block-param.ts @@ -292,7 +292,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - describe.skip.each([ + describe.each([ [405, 'The method does not exist / is not available.'], [429, 'Request is being rate limited.'], ])( @@ -343,7 +343,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe.skip('if the RPC endpoint returns a response that is not 405, 429, 503, or 504', () => { + describe('if the RPC endpoint returns a response that is not 405, 429, 503, or 504', () => { const httpStatus = 500; const errorMessage = `Non-200 status code: '${httpStatus}'`; @@ -391,7 +391,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }); }); - describe.skip.each([503, 504])( + describe.each([503, 504])( 'if the RPC endpoint returns a %d response', (httpStatus) => { const errorMessage = 'Gateway timeout'; @@ -487,7 +487,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe.skip.each(['ETIMEDOUT', 'ECONNRESET'])( + describe.each(['ETIMEDOUT', 'ECONNRESET'])( 'if a %s error is thrown while making the request', (errorCode) => { const error = new Error(errorCode); @@ -579,7 +579,7 @@ export function testsForRpcMethodAssumingNoBlockParam( }, ); - describe.skip('if the RPC endpoint responds with invalid JSON', () => { + describe('if the RPC endpoint responds with invalid JSON', () => { const errorMessage = 'not valid JSON'; it('retries the request up to 5 times until it responds with valid JSON', async () => { diff --git a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts index 5fb8c532b33..7254c5a9670 100644 --- a/packages/network-controller/tests/provider-api-tests/rpc-failover.ts +++ b/packages/network-controller/tests/provider-api-tests/rpc-failover.ts @@ -41,7 +41,7 @@ export function testsForRpcFailoverBehavior({ : maxConsecutiveFailures; describe('if RPC failover functionality is enabled', () => { - it.only(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { + it(`fails over to the provided alternate RPC endpoint after ${maxConsecutiveFailures} unsuccessful attempts`, async () => { await withMockedCommunications({ providerType }, async (primaryComms) => { await withMockedCommunications( { diff --git a/packages/network-controller/tests/provider-api-tests/shared-tests.ts b/packages/network-controller/tests/provider-api-tests/shared-tests.ts index 04e64037e82..0bc264e2911 100644 --- a/packages/network-controller/tests/provider-api-tests/shared-tests.ts +++ b/packages/network-controller/tests/provider-api-tests/shared-tests.ts @@ -32,7 +32,7 @@ export function testsForProviderType(providerType: ProviderType) { // Ethereum JSON-RPC spec: // Infura documentation: describe('methods included in the Ethereum JSON-RPC spec', () => { - describe.skip('methods not handled by middleware', () => { + describe('methods not handled by middleware', () => { const notHandledByMiddleware = [ { name: 'eth_newFilter', numberOfParameters: 1 }, { name: 'eth_getFilterChanges', numberOfParameters: 1 }, @@ -74,7 +74,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe.skip('methods with block hashes in their result', () => { + describe('methods with block hashes in their result', () => { const methodsWithBlockHashInResponse = [ { name: 'eth_getTransactionByHash', numberOfParameters: 1 }, { name: 'eth_getTransactionReceipt', numberOfParameters: 1 }, @@ -132,7 +132,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe.skip('methods that have a param to specify the block', () => { + describe('methods that have a param to specify the block', () => { const supportingBlockParam = [ { name: 'eth_call', @@ -174,7 +174,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe.skip('other methods', () => { + describe('other methods', () => { describe('eth_getTransactionByHash', () => { it("refreshes the block tracker's current block if it is less than the block number that comes back in the response", async () => { const method = 'eth_getTransactionByHash'; @@ -250,7 +250,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe.skip('methods not included in the Ethereum JSON-RPC spec', () => { + describe('methods not included in the Ethereum JSON-RPC spec', () => { describe('methods not handled by middleware', () => { const notHandledByMiddleware = [ { name: 'net_listening', numberOfParameters: 0 }, @@ -270,7 +270,7 @@ export function testsForProviderType(providerType: ProviderType) { }); }); - describe.skip('methods that assume there is no block param', () => { + describe('methods that assume there is no block param', () => { const assumingNoBlockParam = [ { name: 'web3_clientVersion', numberOfParameters: 0 }, { name: 'eth_protocolVersion', numberOfParameters: 0 }, @@ -285,7 +285,7 @@ export function testsForProviderType(providerType: ProviderType) { ); }); - describe.skip('other methods', () => { + describe('other methods', () => { describe('net_version', () => { const networkArgs = { providerType, From 8096cb449bf6eef10a3e1f3f514f4cf4887da2e1 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 14:31:10 -0600 Subject: [PATCH 06/17] wip --- packages/network-controller/package.json | 4 - .../src/NetworkController.ts | 84 ++++++++++--------- .../src/create-auto-managed-network-client.ts | 6 +- .../src/create-network-client.ts | 8 +- packages/network-controller/tests/helpers.ts | 2 +- .../network-controller/tsconfig.build.json | 3 +- .../src/index.ts | 5 +- 7 files changed, 54 insertions(+), 58 deletions(-) diff --git a/packages/network-controller/package.json b/packages/network-controller/package.json index d530fa1db8b..3c106d0b24c 100644 --- a/packages/network-controller/package.json +++ b/packages/network-controller/package.json @@ -69,7 +69,6 @@ "devDependencies": { "@json-rpc-specification/meta-schema": "^1.0.6", "@metamask/auto-changelog": "^3.4.4", - "@metamask/remote-feature-flag-controller": "^1.6.0", "@types/jest": "^27.4.1", "@types/jest-when": "^2.7.3", "@types/lodash": "^4.14.191", @@ -86,9 +85,6 @@ "typedoc-plugin-missing-exports": "^2.0.0", "typescript": "~5.2.2" }, - "peerDependencies": { - "@metamask/remote-feature-flag-controller": "^1.5.0" - }, "engines": { "node": "^18.18 || >=20" }, diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index 3448b05d79d..26d2ceff1bd 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -18,7 +18,6 @@ import { BuiltInNetworkName, } from '@metamask/controller-utils'; import EthQuery from '@metamask/eth-query'; -import type { RemoteFeatureFlagControllerStateChangeEvent } from '@metamask/remote-feature-flag-controller'; import { errorCodes } from '@metamask/rpc-errors'; import { createEventEmitterProxy } from '@metamask/swappable-obj-proxy'; import type { SwappableProxy } from '@metamask/swappable-obj-proxy'; @@ -53,6 +52,7 @@ import type { NetworkClientConfiguration, AdditionalDefaultNetwork, } from './types'; +import { NetworkClient } from './create-network-client'; const debugLog = createModuleLogger(projectLogger, 'NetworkController'); @@ -498,8 +498,6 @@ export type NetworkControllerEvents = | NetworkControllerRpcEndpointDegradedEvent | NetworkControllerRpcEndpointRequestRetriedEvent; -export type AllowedEvents = RemoteFeatureFlagControllerStateChangeEvent; - export type NetworkControllerGetStateAction = ControllerGetStateAction< typeof controllerName, NetworkState @@ -592,14 +590,12 @@ export type NetworkControllerActions = | NetworkControllerRemoveNetworkAction | NetworkControllerUpdateNetworkAction; -export type AllowedActions = never; - export type NetworkControllerMessenger = RestrictedMessenger< typeof controllerName, - NetworkControllerActions | AllowedActions, - NetworkControllerEvents | AllowedEvents, - AllowedActions['type'], - AllowedEvents['type'] + NetworkControllerActions, + NetworkControllerEvents, + never, + never >; /** @@ -1245,14 +1241,34 @@ export class NetworkController extends BaseController< ); } + /** + * Enables the RPC failover functionality. That is, if any RPC endpoints are + * configured with failover URLs, then traffic will automatically be diverted + * to them if those RPC endpoints are unavailable. + */ enableRpcFailover() { this.#updateRpcFailoverEnabled(true); } + /** + * Disables the RPC failover functionality. That is, even if any RPC endpoints + * are configured with failover URLs, then traffic will not automatically be + * diverted to them if those RPC endpoints are unavailable. + */ disableRpcFailover() { this.#updateRpcFailoverEnabled(false); } + /** + * Enables or disables the RPC failover functionality, depending on the + * boolean given. This is done by reconstructing all network clients that were + * originally configured with failover URLs so that those URLs are either + * honored or ignored. Network client IDs will be preserved so as not to + * invalidate state in other controllers. + * + * @param newIsRpcFailoverEnabled - Whether or not to enable or disable the + * RPC failover functionality. + */ #updateRpcFailoverEnabled(newIsRpcFailoverEnabled: boolean) { if (this.#isRpcFailoverEnabled === newIsRpcFailoverEnabled) { return; @@ -1261,41 +1277,27 @@ export class NetworkController extends BaseController< const autoManagedNetworkClientRegistry = this.#ensureAutoManagedNetworkClientRegistryPopulated(); - const infuraAutoManagedNetworkClientRegistry = - autoManagedNetworkClientRegistry[NetworkClientType.Infura]; - for (const networkClientId of knownKeysOf( - infuraAutoManagedNetworkClientRegistry, - )) { - const networkClient = - infuraAutoManagedNetworkClientRegistry[networkClientId]; - if ( - networkClient.configuration.failoverRpcUrls && - networkClient.configuration.failoverRpcUrls.length > 0 - ) { - networkClient.destroy(); - infuraAutoManagedNetworkClientRegistry[networkClientId] = - newIsRpcFailoverEnabled - ? networkClient.withRpcFailoverEnabled() - : networkClient.withRpcFailoverDisabled(); - } - } - - const customAutoManagedNetworkClientRegistry = - autoManagedNetworkClientRegistry[NetworkClientType.Custom]; - for (const networkClientId of knownKeysOf( - customAutoManagedNetworkClientRegistry, + for (const networkClientsById of Object.values( + autoManagedNetworkClientRegistry, )) { - const networkClient = - customAutoManagedNetworkClientRegistry[networkClientId]; - if ( - networkClient.configuration.failoverRpcUrls && - networkClient.configuration.failoverRpcUrls.length > 0 - ) { - networkClient.destroy(); - customAutoManagedNetworkClientRegistry[networkClientId] = - newIsRpcFailoverEnabled + for (const networkClientId of Object.keys(networkClientsById)) { + // Type assertion: We can assume that `networkClientId` is valid here. + const networkClient = + networkClientsById[ + networkClientId as keyof typeof networkClientsById + ]; + if ( + networkClient.configuration.failoverRpcUrls && + networkClient.configuration.failoverRpcUrls.length > 0 + ) { + networkClient.destroy(); + const newNetworkClient = newIsRpcFailoverEnabled ? networkClient.withRpcFailoverEnabled() : networkClient.withRpcFailoverDisabled(); + networkClientsById[ + networkClientId as keyof typeof networkClientsById + ] = newNetworkClient; + } } } diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index cbc1185ba88..d18cc102993 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -69,9 +69,9 @@ const UNINITIALIZED_TARGET = { __UNINITIALIZED__: true }; * @param args.getRpcServiceOptions - Factory for constructing RPC service * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. * @param args.messenger - The network controller messenger. - * @param args.isRpcFailoverEnabled - Whether or not requests sent to the RPC - * endpoint for this network should be automatically diverted to failover RPC - * endpoints (if defined). + * @param args.isRpcFailoverEnabled - Whether or not requests sent to the + * primary RPC endpoint for this network should be automatically diverted to + * provided failover endpoints if the primary is unavailable. * @returns The auto-managed network client. */ export function createAutoManagedNetworkClient< diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index cc0aa7ee274..a832bea9b68 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -57,9 +57,11 @@ export type NetworkClient = { * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. * @param args.messenger - The network controller messenger. * See {@link NetworkControllerOptions.getRpcServiceOptions}. - * @param args.isRpcFailoverEnabled - Whether or not requests sent to the RPC - * endpoint for this network should be automatically diverted to failover RPC - * endpoints (if defined). + * @param args.isRpcFailoverEnabled - Whether or not requests sent to the + * primary RPC endpoint for this network should be automatically diverted to + * provided failover endpoints if the primary is unavailable. This effectively + * causes the `failoverRpcUrls` property of the network client configuration + * to be honored or ignored. * @returns The network client. */ export function createNetworkClient({ diff --git a/packages/network-controller/tests/helpers.ts b/packages/network-controller/tests/helpers.ts index 09ffb1938f2..cf06125ad04 100644 --- a/packages/network-controller/tests/helpers.ts +++ b/packages/network-controller/tests/helpers.ts @@ -90,7 +90,7 @@ export function buildNetworkControllerMessenger( return messenger.getRestricted({ name: 'NetworkController', allowedActions: [], - allowedEvents: ['RemoteFeatureFlagController:stateChange'], + allowedEvents: [], }); } diff --git a/packages/network-controller/tsconfig.build.json b/packages/network-controller/tsconfig.build.json index 31f78014177..c054df5ef38 100644 --- a/packages/network-controller/tsconfig.build.json +++ b/packages/network-controller/tsconfig.build.json @@ -9,8 +9,7 @@ { "path": "../base-controller/tsconfig.build.json" }, { "path": "../controller-utils/tsconfig.build.json" }, { "path": "../eth-json-rpc-provider/tsconfig.build.json" }, - { "path": "../json-rpc-engine/tsconfig.build.json" }, - { "path": "../remote-feature-flag-controller/tsconfig.build.json" } + { "path": "../json-rpc-engine/tsconfig.build.json" } ], "include": ["../../types", "./src"] } diff --git a/packages/remote-feature-flag-controller/src/index.ts b/packages/remote-feature-flag-controller/src/index.ts index c43f2dce998..2c5e2cd8025 100644 --- a/packages/remote-feature-flag-controller/src/index.ts +++ b/packages/remote-feature-flag-controller/src/index.ts @@ -1,8 +1,5 @@ export { RemoteFeatureFlagController } from './remote-feature-flag-controller'; -export type { - RemoteFeatureFlagControllerMessenger, - RemoteFeatureFlagControllerStateChangeEvent, -} from './remote-feature-flag-controller'; +export type { RemoteFeatureFlagControllerMessenger } from './remote-feature-flag-controller'; export { ClientType, DistributionType, From 7a3ed49b7d97d16c49718161542ab600ec25aa6a Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 14:35:05 -0600 Subject: [PATCH 07/17] done --- jest.config.packages.js | 2 +- packages/network-controller/CHANGELOG.md | 9 ++++++- .../tests/NetworkController.test.ts | 26 +++++++++++++------ packages/network-controller/tests/helpers.ts | 10 +++---- .../tests/provider-api-tests/helpers.ts | 8 +++--- .../tests/provider-api-tests/shared-tests.ts | 1 - packages/network-controller/tsconfig.json | 3 --- 7 files changed, 34 insertions(+), 25 deletions(-) diff --git a/jest.config.packages.js b/jest.config.packages.js index 1ec5510e276..fd5e2eb5e94 100644 --- a/jest.config.packages.js +++ b/jest.config.packages.js @@ -180,7 +180,7 @@ module.exports = { // testRunner: "jest-circus/runner", // Default timeout of a test in milliseconds. - testTimeout: 3000, + testTimeout: 30000, // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href // testURL: "http://localhost", diff --git a/packages/network-controller/CHANGELOG.md b/packages/network-controller/CHANGELOG.md index 46d280e6eed..0433e71af06 100644 --- a/packages/network-controller/CHANGELOG.md +++ b/packages/network-controller/CHANGELOG.md @@ -9,7 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add optional `rpcFailoverEnabled` option to `NetworkController` constructor (`false` by default) ([#5668](https://github.com/MetaMask/core/pull/5668)) +- Add optional `rpcFailoverEnabled` option to NetworkController constructor (`false` by default) ([#5668](https://github.com/MetaMask/core/pull/5668)) +- Add `enableRpcFailover` and `disableRpcFailover` methods to NetworkController ([#5668](https://github.com/MetaMask/core/pull/5668)) + +### Changed + +- Disable the RPC failover behavior by default ([#5668](https://github.com/MetaMask/core/pull/5668)) + - You are free to set the `failoverUrls` property on an RPC endpoint, but it won't have any effect + - To enable this behavior, either pass `rpcFailoverEnabled: true` to the constructor or call `enableRpcFailover` after initialization ## [23.2.0] diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index b0dcc0400c6..982fccfda75 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -16,12 +16,11 @@ import { import { rpcErrors } from '@metamask/rpc-errors'; import type { Hex } from '@metamask/utils'; import assert from 'assert'; -import { produceWithPatches, type Patch } from 'immer'; +import type { Patch } from 'immer'; import { when, resetAllWhenMocks } from 'jest-when'; import { inspect, isDeepStrictEqual, promisify } from 'util'; import { v4 as uuidV4 } from 'uuid'; -import type { RootMessenger } from './helpers'; import { buildAddNetworkCustomRpcEndpointFields, buildAddNetworkFields, @@ -1327,7 +1326,10 @@ describe('NetworkController', () => { { messenger, }: { - messenger: RootMessenger; + messenger: Messenger< + NetworkControllerActions, + NetworkControllerEvents + >; }, args: Parameters, ): ReturnType => @@ -3264,7 +3266,13 @@ describe('NetworkController', () => { ], [ 'NetworkController:getNetworkConfigurationByChainId', - ({ messenger, chainId }: { messenger: RootMessenger; chainId: Hex }) => + ({ + messenger, + chainId, + }: { + messenger: Messenger; + chainId: Hex; + }) => messenger.call( 'NetworkController:getNetworkConfigurationByChainId', chainId, @@ -3377,7 +3385,7 @@ describe('NetworkController', () => { messenger, networkClientId, }: { - messenger: RootMessenger; + messenger: Messenger; networkClientId: NetworkClientId; }) => messenger.call( @@ -5354,6 +5362,8 @@ describe('NetworkController', () => { const infuraRpcEndpoint: InfuraRpcEndpoint = { failoverUrls: ['https://failover.endpoint'], networkClientId: infuraNetworkType, + // ESLint is mistaken here. + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions url: `https://${infuraNetworkType}.infura.io/v3/{infuraProjectId}`, type: RpcEndpointType.Infura, }; @@ -15091,7 +15101,7 @@ type WithControllerCallback = ({ controller, }: { controller: NetworkController; - messenger: RootMessenger; + messenger: Messenger; networkControllerMessenger: NetworkControllerMessenger; }) => Promise | ReturnValue; @@ -15270,7 +15280,7 @@ async function waitForPublishedEvents({ // do nothing }, }: { - messenger: RootMessenger; + messenger: Messenger; eventType: E['type']; count?: number; filter?: (payload: E['payload']) => boolean; @@ -15401,7 +15411,7 @@ async function waitForStateChanges({ operation, beforeResolving, }: { - messenger: RootMessenger; + messenger: Messenger; propertyPath?: string[]; count?: number; wait?: number; diff --git a/packages/network-controller/tests/helpers.ts b/packages/network-controller/tests/helpers.ts index cf06125ad04..30def58c9ef 100644 --- a/packages/network-controller/tests/helpers.ts +++ b/packages/network-controller/tests/helpers.ts @@ -25,8 +25,6 @@ import type { AutoManagedNetworkClient } from '../src/create-auto-managed-networ import type { AddNetworkCustomRpcEndpointFields, AddNetworkFields, - AllowedActions, - AllowedEvents, CustomRpcEndpoint, InfuraRpcEndpoint, NetworkControllerActions, @@ -42,8 +40,8 @@ import type { import { NetworkClientType } from '../src/types'; export type RootMessenger = Messenger< - NetworkControllerActions | AllowedActions, - NetworkControllerEvents | AllowedEvents + NetworkControllerActions, + NetworkControllerEvents >; /** @@ -75,7 +73,7 @@ export const TESTNET = { * @returns The messenger. */ export function buildRootMessenger(): RootMessenger { - return new Messenger(); + return new Messenger(); } /** @@ -104,7 +102,7 @@ export function buildNetworkControllerMessenger( * requests. * @returns The fake network client. */ -export function buildFakeNetworkClient({ +function buildFakeNetworkClient({ configuration, providerStubs = [], }: { diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 29cdea4b2a8..3ee5757f033 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -168,7 +168,7 @@ function mockRpcCall({ ? `/v3/${MOCK_INFURA_PROJECT_ID}` : '/'; - debug('Nock Mocking Request:', { + debug('Mocking request:', { url, method, params, @@ -201,7 +201,6 @@ function mockRpcCall({ // eslint-disable-next-line @typescript-eslint/no-explicit-any return nockRequest.reply(httpStatus, (_, requestBody: any) => { if (typeof completeResponse === 'string') { - debug('Nock Returning Response', completeResponse); return completeResponse; } @@ -212,12 +211,11 @@ function mockRpcCall({ completeResponse.id = response.id; } } - debug('Nock Returning Response', completeResponse); + debug('Nock returning Response', completeResponse); return completeResponse; }); } - - throw new Error('No reply specified'); + return nockRequest; } type MockBlockTrackerRequestOptions = { diff --git a/packages/network-controller/tests/provider-api-tests/shared-tests.ts b/packages/network-controller/tests/provider-api-tests/shared-tests.ts index 0bc264e2911..d49b05c2c7e 100644 --- a/packages/network-controller/tests/provider-api-tests/shared-tests.ts +++ b/packages/network-controller/tests/provider-api-tests/shared-tests.ts @@ -121,7 +121,6 @@ export function testsForProviderType(providerType: ProviderType) { ]; assumingNoBlockParam .concat(blockParamIgnored) - .slice(0, 1) .forEach(({ name, numberOfParameters }) => describe(`method name: ${name}`, () => { testsForRpcMethodAssumingNoBlockParam(name, { diff --git a/packages/network-controller/tsconfig.json b/packages/network-controller/tsconfig.json index 8362d23ec4f..c6a988886f9 100644 --- a/packages/network-controller/tsconfig.json +++ b/packages/network-controller/tsconfig.json @@ -16,9 +16,6 @@ }, { "path": "../json-rpc-engine" - }, - { - "path": "../remote-feature-flag-controller" } ], "include": ["../../types", "../../tests", "./src", "./tests"] From 08d69073642cebfa1f16438b5c342b172002e074 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sat, 19 Apr 2025 22:03:59 -0600 Subject: [PATCH 08/17] NetworkController: Add way to customize block tracker Add an optional `getBlockTrackerOptions` argument to NetworkController, which can be used to customize the block tracker for a particular network (or all networks if desired). This is particularly useful when finetuning the behavior of the RPC failover logic. --- packages/network-controller/CHANGELOG.md | 4 ++ .../src/NetworkController.ts | 31 ++++++++-- ...create-auto-managed-network-client.test.ts | 9 +++ .../src/create-auto-managed-network-client.ts | 11 ++++ .../src/create-network-client.ts | 59 ++++++++++++++++--- .../tests/NetworkController.test.ts | 58 ++++++++++++++++++ .../tests/provider-api-tests/helpers.ts | 4 ++ 7 files changed, 162 insertions(+), 14 deletions(-) diff --git a/packages/network-controller/CHANGELOG.md b/packages/network-controller/CHANGELOG.md index 22c9414f03a..aa1968361d9 100644 --- a/packages/network-controller/CHANGELOG.md +++ b/packages/network-controller/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add optional `getBlockTrackerOptions` argument to NetworkController constructor ([#XXXX](https://github.com/MetaMask/core/pull/XXXX)) + ## [23.2.0] ### Added diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index c8ab5d227f3..5cb6618be78 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -52,6 +52,7 @@ import type { NetworkClientConfiguration, AdditionalDefaultNetwork, } from './types'; +import { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; const debugLog = createModuleLogger(projectLogger, 'NetworkController'); @@ -621,16 +622,23 @@ export type NetworkControllerOptions = { */ log?: Logger; /** - * A function that can be used to customize the options passed to a - * RPC service constructed for an RPC endpoint. The object that the function - * should return is the same as {@link RpcServiceOptions}, except that - * `failoverService` and `endpointUrl` are not accepted (as they are filled in - * automatically). + * A function that can be used to customize a RPC service constructed for an + * RPC endpoint. The function takes the URL of the endpoint and should return + * an object with type {@link RpcServiceOptions}, minus `failoverService` + * and `endpointUrl` (as they are filled in automatically). */ getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; - + /** + * A function that can be used to customize a block tracker constructed for an + * RPC endpoint. The function takes the URL of the endpoint and should return + * an object of type {@link PollingBlockTrackerOptions}, minus `provider` (as + * it is filled in automatically). + */ + getBlockTrackerOptions?: ( + rpcEndpointUrl: string, + ) => Omit; /** * An array of Hex Chain IDs representing the additional networks to be included as default. */ @@ -1080,6 +1088,11 @@ export class NetworkController extends BaseController< readonly #getRpcServiceOptions: NetworkControllerOptions['getRpcServiceOptions']; + readonly #getBlockTrackerOptions: Exclude< + NetworkControllerOptions['getBlockTrackerOptions'], + undefined + >; + #networkConfigurationsByNetworkClientId: Map< NetworkClientId, NetworkConfiguration @@ -1097,6 +1110,7 @@ export class NetworkController extends BaseController< infuraProjectId, log, getRpcServiceOptions, + getBlockTrackerOptions = () => ({}), additionalDefaultNetworks, } = options; const initialState = { @@ -1131,6 +1145,7 @@ export class NetworkController extends BaseController< this.#infuraProjectId = infuraProjectId; this.#log = log; this.#getRpcServiceOptions = getRpcServiceOptions; + this.#getBlockTrackerOptions = getBlockTrackerOptions; this.#previouslySelectedNetworkClientId = this.state.selectedNetworkClientId; @@ -2638,6 +2653,7 @@ export class NetworkController extends BaseController< ticker: networkFields.nativeCurrency, }, getRpcServiceOptions: this.#getRpcServiceOptions, + getBlockTrackerOptions: this.#getBlockTrackerOptions, messenger: this.messagingSystem, }); } else { @@ -2652,6 +2668,7 @@ export class NetworkController extends BaseController< ticker: networkFields.nativeCurrency, }, getRpcServiceOptions: this.#getRpcServiceOptions, + getBlockTrackerOptions: this.#getBlockTrackerOptions, messenger: this.messagingSystem, }); } @@ -2812,6 +2829,7 @@ export class NetworkController extends BaseController< ticker: networkConfiguration.nativeCurrency, }, getRpcServiceOptions: this.#getRpcServiceOptions, + getBlockTrackerOptions: this.#getBlockTrackerOptions, messenger: this.messagingSystem, }), ] as const; @@ -2827,6 +2845,7 @@ export class NetworkController extends BaseController< ticker: networkConfiguration.nativeCurrency, }, getRpcServiceOptions: this.#getRpcServiceOptions, + getBlockTrackerOptions: this.#getBlockTrackerOptions, messenger: this.messagingSystem, }), ] as const; diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index 4d9d6e8cceb..610d6b6ef8d 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -44,6 +44,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -59,6 +60,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); }).not.toThrow(); @@ -71,6 +73,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -117,6 +120,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -166,6 +170,7 @@ describe('createAutoManagedNetworkClient', () => { const { provider } = createAutoManagedNetworkClient({ networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -196,6 +201,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -253,6 +259,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -323,6 +330,7 @@ describe('createAutoManagedNetworkClient', () => { const { blockTracker } = createAutoManagedNetworkClient({ networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); @@ -363,6 +371,7 @@ describe('createAutoManagedNetworkClient', () => { fetch, btoa, }), + getBlockTrackerOptions: () => ({}), messenger: getNetworkControllerMessenger(), }); // Start the block tracker diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index 9fde14a2f5a..7daabbe6783 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -1,3 +1,4 @@ +import { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; import type { NetworkClient } from './create-network-client'; import { createNetworkClient } from './create-network-client'; import type { NetworkControllerMessenger } from './NetworkController'; @@ -66,6 +67,8 @@ const UNINITIALIZED_TARGET = { __UNINITIALIZED__: true }; * used to instantiate the network client when it is needed. * @param args.getRpcServiceOptions - Factory for constructing RPC service * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. + * @param args.getBlockTrackerOptions - Factory for constructing block tracker + * options. See {@link NetworkControllerOptions.getBlockTrackerOptions}. * @param args.messenger - The network controller messenger. * @returns The auto-managed network client. */ @@ -74,12 +77,16 @@ export function createAutoManagedNetworkClient< >({ networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }: { networkClientConfiguration: Configuration; getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; + getBlockTrackerOptions: ( + rpcEndpointUrl: string, + ) => Omit; messenger: NetworkControllerMessenger; }): AutoManagedNetworkClient { let networkClient: NetworkClient | undefined; @@ -95,6 +102,7 @@ export function createAutoManagedNetworkClient< networkClient ??= createNetworkClient({ configuration: networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }); if (networkClient === undefined) { @@ -136,6 +144,7 @@ export function createAutoManagedNetworkClient< networkClient ??= createNetworkClient({ configuration: networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }); const { provider } = networkClient; @@ -156,6 +165,7 @@ export function createAutoManagedNetworkClient< networkClient ??= createNetworkClient({ configuration: networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }); if (networkClient === undefined) { @@ -197,6 +207,7 @@ export function createAutoManagedNetworkClient< networkClient ??= createNetworkClient({ configuration: networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }); const { blockTracker } = networkClient; diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index 604e94a02d7..a8e1784bfbe 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -1,6 +1,9 @@ import type { InfuraNetworkType } from '@metamask/controller-utils'; import { ChainId } from '@metamask/controller-utils'; -import { PollingBlockTracker } from '@metamask/eth-block-tracker'; +import { + PollingBlockTracker, + PollingBlockTrackerOptions, +} from '@metamask/eth-block-tracker'; import { createInfuraMiddleware } from '@metamask/eth-json-rpc-infura'; import { createBlockCacheMiddleware, @@ -55,19 +58,24 @@ export type NetworkClient = { * @param args.configuration - The network configuration. * @param args.getRpcServiceOptions - Factory for constructing RPC service * options. See {@link NetworkControllerOptions.getRpcServiceOptions}. + * @param args.getBlockTrackerOptions - Factory for constructing block tracker + * options. See {@link NetworkControllerOptions.getBlockTrackerOptions}. * @param args.messenger - The network controller messenger. - * See {@link NetworkControllerOptions.getRpcServiceOptions}. * @returns The network client. */ export function createNetworkClient({ configuration, getRpcServiceOptions, + getBlockTrackerOptions, messenger, }: { configuration: NetworkClientConfiguration; getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; + getBlockTrackerOptions: ( + rpcEndpointUrl: string, + ) => Omit; messenger: NetworkControllerMessenger; }): NetworkClient { const primaryEndpointUrl = @@ -124,12 +132,10 @@ export function createNetworkClient({ const rpcProvider = providerFromMiddleware(rpcApiMiddleware); - const blockTrackerOpts = - process.env.IN_TEST && configuration.type === NetworkClientType.Custom - ? { pollingInterval: SECOND } - : {}; - const blockTracker = new PollingBlockTracker({ - ...blockTrackerOpts, + const blockTracker = createBlockTracker({ + networkClientType: configuration.type, + endpointUrl: primaryEndpointUrl, + getOptions: getBlockTrackerOptions, provider: rpcProvider, }); @@ -162,6 +168,43 @@ export function createNetworkClient({ return { configuration, provider, blockTracker, destroy }; } +/** + * Create the block tracker for the network. + * + * @param args - The arguments. + * @param args.networkClientType - The type of the network client ("infura" or + * "custom"). + * @param args.endpointUrl - The URL of the endpoint. + * @param args.getOptions - Factory for the block tracker options. + * @param args.provider - The EIP-1193 provider for the network's JSON-RPC + * middleware stack. + * @returns The created block tracker. + */ +function createBlockTracker({ + networkClientType, + endpointUrl, + getOptions, + provider, +}: { + networkClientType: NetworkClientType; + endpointUrl: string; + getOptions: ( + rpcEndpointUrl: string, + ) => Omit; + provider: SafeEventEmitterProvider; +}) { + const testOptions = + process.env.IN_TEST && networkClientType === NetworkClientType.Custom + ? { pollingInterval: SECOND } + : {}; + + return new PollingBlockTracker({ + ...testOptions, + ...getOptions(endpointUrl), + provider, + }); +} + /** * Create middleware for infura. * diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index e40de31fe2b..f094c7cdb53 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -3605,6 +3605,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -3624,6 +3627,7 @@ describe('NetworkController', () => { }), infuraProjectId, getRpcServiceOptions, + getBlockTrackerOptions, }, ({ controller, networkControllerMessenger }) => { const defaultRpcEndpoint: InfuraRpcEndpoint = { @@ -3672,6 +3676,7 @@ describe('NetworkController', () => { type: NetworkClientType.Infura, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -3686,6 +3691,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -3700,6 +3706,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -5040,6 +5047,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -5061,6 +5071,7 @@ describe('NetworkController', () => { }, infuraProjectId, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { const infuraRpcEndpoint: InfuraRpcEndpoint = { @@ -5093,6 +5104,7 @@ describe('NetworkController', () => { type: NetworkClientType.Infura, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); @@ -5267,6 +5279,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -5288,6 +5303,7 @@ describe('NetworkController', () => { }, infuraProjectId: 'some-infura-project-id', getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { const [rpcEndpoint1, rpcEndpoint2] = [ @@ -5320,6 +5336,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); expect( @@ -5333,6 +5350,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); @@ -6252,6 +6270,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -6272,6 +6293,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockReturnValue(buildFakeClient()); @@ -6298,6 +6320,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); const networkConfigurationsByNetworkClientId = @@ -7110,6 +7133,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -7130,6 +7156,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { await controller.updateNetwork('0x1337', { @@ -7161,6 +7188,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -7175,6 +7203,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -8094,6 +8123,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -8114,6 +8146,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -8153,6 +8186,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); expect( @@ -9256,6 +9290,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -9276,6 +9313,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -9310,6 +9348,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); expect( @@ -9323,6 +9362,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); @@ -9964,6 +10004,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -9984,6 +10027,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -10023,6 +10067,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); expect(createAutoManagedNetworkClientSpy).toHaveBeenCalledWith({ @@ -10034,6 +10079,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); @@ -10692,6 +10738,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -10713,6 +10762,7 @@ describe('NetworkController', () => { }, infuraProjectId: 'some-infura-project-id', getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation( @@ -10751,6 +10801,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); expect( @@ -10764,6 +10815,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); @@ -11387,6 +11439,9 @@ describe('NetworkController', () => { maxConsecutiveFailures: 10, }, }); + const getBlockTrackerOptions = () => ({ + pollingInterval: 2000, + }); await withController( { @@ -11407,6 +11462,7 @@ describe('NetworkController', () => { selectedNetworkClientId: 'ZZZZ-ZZZZ-ZZZZ-ZZZZ', }, getRpcServiceOptions, + getBlockTrackerOptions, }, async ({ controller, networkControllerMessenger }) => { createNetworkClientMock.mockImplementation(({ configuration }) => { @@ -11439,6 +11495,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); @@ -11453,6 +11510,7 @@ describe('NetworkController', () => { type: NetworkClientType.Custom, }, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }, ); diff --git a/packages/network-controller/tests/provider-api-tests/helpers.ts b/packages/network-controller/tests/provider-api-tests/helpers.ts index 59bd53097c2..ba1d2c8c54c 100644 --- a/packages/network-controller/tests/provider-api-tests/helpers.ts +++ b/packages/network-controller/tests/provider-api-tests/helpers.ts @@ -311,6 +311,7 @@ export type MockOptions = { customChainId?: Hex; customTicker?: string; getRpcServiceOptions?: NetworkControllerOptions['getRpcServiceOptions']; + getBlockTrackerOptions?: NetworkControllerOptions['getBlockTrackerOptions']; expectedHeaders?: Record; messenger?: RootMessenger; }; @@ -472,6 +473,7 @@ export async function waitForPromiseToBeFulfilledAfterRunningAllTimers( * @param options.customTicker - The ticker of the custom RPC endpoint, assuming * that `providerType` is "custom" (default: "ETH"). * @param options.getRpcServiceOptions - RPC service options factory. + * @param options.getBlockTrackerOptions - Block tracker options factory. * @param options.messenger - The root messenger to use in tests. * @param fn - A function which will be called with an object that allows * interaction with the network client. @@ -486,6 +488,7 @@ export async function withNetworkClient( customChainId = '0x1', customTicker = 'ETH', getRpcServiceOptions = () => ({ fetch, btoa }), + getBlockTrackerOptions = () => ({}), messenger = buildRootMessenger(), }: MockOptions, // TODO: Replace `any` with type @@ -537,6 +540,7 @@ export async function withNetworkClient( const networkClient = createNetworkClient({ configuration: networkClientConfiguration, getRpcServiceOptions, + getBlockTrackerOptions, messenger: networkControllerMessenger, }); /* eslint-disable-next-line n/no-process-env */ From 3044a1a5c47b214f041f1cc59769e424e1424f7c Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 12:36:47 -0600 Subject: [PATCH 09/17] Fix lint --- packages/network-controller/src/NetworkController.ts | 3 +-- .../src/create-auto-managed-network-client.ts | 3 ++- packages/network-controller/src/create-network-client.ts | 2 +- .../tests/provider-api-tests/block-hash-in-response.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index 878e348629c..212ec0577c5 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -17,6 +17,7 @@ import { BUILT_IN_NETWORKS, BuiltInNetworkName, } from '@metamask/controller-utils'; +import type { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; import EthQuery from '@metamask/eth-query'; import { errorCodes } from '@metamask/rpc-errors'; import { createEventEmitterProxy } from '@metamask/swappable-obj-proxy'; @@ -52,8 +53,6 @@ import type { NetworkClientConfiguration, AdditionalDefaultNetwork, } from './types'; -import { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; -import { NetworkClient } from './create-network-client'; const debugLog = createModuleLogger(projectLogger, 'NetworkController'); diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index d3c59b388a1..bee7b890a97 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -1,4 +1,5 @@ -import { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; +import type { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; + import type { NetworkClient } from './create-network-client'; import { createNetworkClient } from './create-network-client'; import type { NetworkControllerMessenger } from './NetworkController'; diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index 27095811f0b..998c629ab6f 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -1,6 +1,6 @@ import type { InfuraNetworkType } from '@metamask/controller-utils'; import { ChainId } from '@metamask/controller-utils'; -import { +import type { PollingBlockTracker, PollingBlockTrackerOptions, } from '@metamask/eth-block-tracker'; diff --git a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts index 576896ace64..95fc8c1f68b 100644 --- a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts +++ b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts @@ -6,8 +6,8 @@ import { withMockedCommunications, withNetworkClient, } from './helpers'; -import { NetworkClientType } from '../../src/types'; import { testsForRpcFailoverBehavior } from './rpc-failover'; +import { NetworkClientType } from '../../src/types'; type TestsForRpcMethodThatCheckForBlockHashInResponseOptions = { providerType: ProviderType; From 1489ac28be347c2d2d9154e4b56edeef58a3c4d7 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 12:55:32 -0600 Subject: [PATCH 10/17] Fix yarn.lock --- yarn.lock | 3 --- 1 file changed, 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index ff7411a4f9a..767928188d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3818,7 +3818,6 @@ __metadata: "@metamask/eth-json-rpc-provider": "npm:^4.1.8" "@metamask/eth-query": "npm:^4.0.0" "@metamask/json-rpc-engine": "npm:^10.0.3" - "@metamask/remote-feature-flag-controller": "npm:^1.6.0" "@metamask/rpc-errors": "npm:^7.0.2" "@metamask/swappable-obj-proxy": "npm:^2.3.0" "@metamask/utils": "npm:^11.2.0" @@ -3844,8 +3843,6 @@ __metadata: typescript: "npm:~5.2.2" uri-js: "npm:^4.4.1" uuid: "npm:^8.3.2" - peerDependencies: - "@metamask/remote-feature-flag-controller": ^1.5.0 languageName: unknown linkType: soft From cda9282607229862276119265ad85da0c17ec01e Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 12:56:36 -0600 Subject: [PATCH 11/17] Fix changelog --- packages/network-controller/CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/network-controller/CHANGELOG.md b/packages/network-controller/CHANGELOG.md index 34c65a5b403..5b79b8fb98b 100644 --- a/packages/network-controller/CHANGELOG.md +++ b/packages/network-controller/CHANGELOG.md @@ -9,13 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add optional `getBlockTrackerOptions` argument to NetworkController constructor ([#XXXX](https://github.com/MetaMask/core/pull/XXXX)) -- Add optional `rpcFailoverEnabled` option to NetworkController constructor (`false` by default) ([#5668](https://github.com/MetaMask/core/pull/5668)) -- Add `enableRpcFailover` and `disableRpcFailover` methods to NetworkController ([#5668](https://github.com/MetaMask/core/pull/5668)) +- Add optional `getBlockTrackerOptions` argument to NetworkController constructor ([#5679](https://github.com/MetaMask/core/pull/5679)) +- Add optional `rpcFailoverEnabled` option to NetworkController constructor (`false` by default) ([#5679](https://github.com/MetaMask/core/pull/5679)) +- Add `enableRpcFailover` and `disableRpcFailover` methods to NetworkController ([#5679](https://github.com/MetaMask/core/pull/5679)) ### Changed -- Disable the RPC failover behavior by default ([#5668](https://github.com/MetaMask/core/pull/5668)) +- Disable the RPC failover behavior by default ([#5679](https://github.com/MetaMask/core/pull/5679)) - You are free to set the `failoverUrls` property on an RPC endpoint, but it won't have any effect - To enable this behavior, either pass `rpcFailoverEnabled: true` to the constructor or call `enableRpcFailover` after initialization From 22dc1bc4f621876ed03f0e619eebf29c6f992ea3 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 13:06:28 -0600 Subject: [PATCH 12/17] Fix build --- packages/network-controller/src/create-network-client.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/network-controller/src/create-network-client.ts b/packages/network-controller/src/create-network-client.ts index 998c629ab6f..c6388ae3c18 100644 --- a/packages/network-controller/src/create-network-client.ts +++ b/packages/network-controller/src/create-network-client.ts @@ -1,9 +1,7 @@ import type { InfuraNetworkType } from '@metamask/controller-utils'; import { ChainId } from '@metamask/controller-utils'; -import type { - PollingBlockTracker, - PollingBlockTrackerOptions, -} from '@metamask/eth-block-tracker'; +import type { PollingBlockTrackerOptions } from '@metamask/eth-block-tracker'; +import { PollingBlockTracker } from '@metamask/eth-block-tracker'; import { createInfuraMiddleware } from '@metamask/eth-json-rpc-infura'; import { createBlockCacheMiddleware, From 66f06159a21e81b2f83c3b2c5db97aa7f25e08f5 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 13:19:53 -0600 Subject: [PATCH 13/17] Fix test coverage --- .../src/create-auto-managed-network-client.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index bee7b890a97..1897d69e610 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -108,6 +108,9 @@ export function createAutoManagedNetworkClient< isRpcFailoverEnabled, }); + // We don't need to test this; this itself is to catch an edge case where + // `createNetworkClient` is mocked improperly in a test + /* istanbul ignore next */ if (networkClient === undefined) { throw new Error( "It looks like `createNetworkClient` didn't return anything. Perhaps it's being mocked?", From 37ff8d827dc988a41dde6cbadb3db5d5fc218ef5 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 22:54:37 -0600 Subject: [PATCH 14/17] (Goes with enable/disable) Fix proxies after enable/disable --- .../src/NetworkController.ts | 27 +- .../tests/NetworkController.test.ts | 456 +++++++++++++++++- packages/network-controller/tests/helpers.ts | 2 +- 3 files changed, 473 insertions(+), 12 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index 212ec0577c5..641a5bd37e6 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -1312,6 +1312,8 @@ export class NetworkController extends BaseController< networkClientsById[ networkClientId as keyof typeof networkClientsById ] = newNetworkClient; + + this.#setProxies(newNetworkClient); } } } @@ -3028,25 +3030,32 @@ export class NetworkController extends BaseController< updateState?.(state); }); + const { providerProxy } = this.#setProxies(this.#autoManagedNetworkClient); + + this.#ethQuery = new EthQuery(providerProxy); + } + + #setProxies( + networkClient: AutoManagedNetworkClient, + ) { if (this.#providerProxy) { - this.#providerProxy.setTarget(this.#autoManagedNetworkClient.provider); + this.#providerProxy.setTarget(networkClient.provider); } else { - this.#providerProxy = createEventEmitterProxy( - this.#autoManagedNetworkClient.provider, - ); + this.#providerProxy = createEventEmitterProxy(networkClient.provider); } if (this.#blockTrackerProxy) { - this.#blockTrackerProxy.setTarget( - this.#autoManagedNetworkClient.blockTracker, - ); + this.#blockTrackerProxy.setTarget(networkClient.blockTracker); } else { this.#blockTrackerProxy = createEventEmitterProxy( - this.#autoManagedNetworkClient.blockTracker, + networkClient.blockTracker, { eventFilter: 'skipInternal' }, ); } - this.#ethQuery = new EthQuery(this.#providerProxy); + return { + providerProxy: this.#providerProxy, + blockTrackerProxy: this.#blockTrackerProxy, + }; } } diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index 834d96b582a..a1ee46a68c0 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -13,6 +13,7 @@ import { NetworkType, toHex, } from '@metamask/controller-utils'; +import { PollingBlockTracker } from '@metamask/eth-block-tracker'; import { rpcErrors } from '@metamask/rpc-errors'; import type { Hex } from '@metamask/utils'; import assert from 'assert'; @@ -37,7 +38,6 @@ import { INFURA_NETWORKS, TESTNET, } from './helpers'; -import { FakeBlockTracker } from '../../../tests/fake-block-tracker'; import type { FakeProviderStub } from '../../../tests/fake-provider'; import { FakeProvider } from '../../../tests/fake-provider'; import { NetworkStatus } from '../src/constants'; @@ -69,6 +69,7 @@ import { } from '../src/NetworkController'; import type { NetworkClientConfiguration, Provider } from '../src/types'; import { NetworkClientType } from '../src/types'; +import { flushPromises } from '../../../tests/helpers'; jest.mock('../src/create-network-client'); @@ -699,6 +700,7 @@ describe('NetworkController', () => { autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); + createNetworkClientMock.mockReturnValue(buildFakeClient()); controller.enableRpcFailover(); @@ -715,6 +717,230 @@ describe('NetworkController', () => { }, ); }); + + it('destroys only the existing network clients whose RPC endpoints have configured failover URLs', async () => { + await withController( + { + isRpcFailoverEnabled: false, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'destroy'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); + createNetworkClientMock.mockReturnValue(buildFakeClient()); + + controller.enableRpcFailover(); + await flushPromises(); + + expect(autoManagedNetworkClients).toHaveLength(3); + expect(autoManagedNetworkClients[0].destroy).not.toHaveBeenCalled(); + expect(autoManagedNetworkClients[1].destroy).toHaveBeenCalled(); + expect(autoManagedNetworkClients[2].destroy).toHaveBeenCalled(); + }, + ); + }); + + it('updates the provider proxy to point to the new selected network client if it was recreated', async () => { + await withController( + { + isRpcFailoverEnabled: false, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + '0x100': buildCustomNetworkConfiguration({ + chainId: '0x100', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network', + failoverUrls: ['https://failover.endpoint'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const fakeProviders = [ + buildFakeProvider([ + { + request: { + method: 'test_method', + }, + response: { + result: 'response from provider 1', + }, + }, + ]), + buildFakeProvider([ + { + request: { + method: 'test_method', + }, + response: { + result: 'response from provider 2', + }, + }, + ]), + ]; + const fakeNetworkClients = [ + buildFakeClient(fakeProviders[0]), + buildFakeClient(fakeProviders[1]), + ]; + let i = 0; + createNetworkClientMock.mockImplementation(({ configuration }) => { + if (i === 0) { + i += 1; + return fakeNetworkClients[0]; + } else if (i === 1) { + i += 1; + return fakeNetworkClients[1]; + } + throw new Error( + `Unknown network client configuration ${JSON.stringify( + configuration, + )}`, + ); + }); + await controller.initializeProvider(); + + controller.enableRpcFailover(); + await flushPromises(); + + const { provider } = controller.getProviderAndBlockTracker(); + assert(provider, 'Provider is somehow unset'); + const result = await provider.request({ + id: '1', + jsonrpc: '2.0', + method: 'test_method', + }); + expect(result).toBe('response from provider 2'); + }, + ); + }); + + it('updates the block tracker proxy to point to the new selected network client if it was recreated', async () => { + await withController( + { + isRpcFailoverEnabled: false, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + '0x100': buildCustomNetworkConfiguration({ + chainId: '0x100', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network', + failoverUrls: ['https://failover.endpoint'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const fakeProviders = [ + buildFakeProvider([ + { + request: { + method: 'eth_blockNumber', + }, + response: { + result: '0x100', + }, + }, + ]), + buildFakeProvider([ + { + request: { + method: 'eth_blockNumber', + }, + response: { + result: '0x200', + }, + }, + ]), + ]; + const fakeNetworkClients = [ + buildFakeClient(fakeProviders[0]), + buildFakeClient(fakeProviders[1]), + ]; + let i = 0; + createNetworkClientMock.mockImplementation(({ configuration }) => { + if (i === 0) { + i += 1; + return fakeNetworkClients[0]; + } else if (i === 1) { + i += 1; + return fakeNetworkClients[1]; + } + throw new Error( + `Unknown network client configuration ${JSON.stringify( + configuration, + )}`, + ); + }); + await controller.initializeProvider(); + + controller.enableRpcFailover(); + await flushPromises(); + + const { blockTracker } = controller.getProviderAndBlockTracker(); + assert(blockTracker, 'Block tracker is somehow unset'); + const latestBlockNumber = await blockTracker.getLatestBlock(); + expect(latestBlockNumber).toBe('0x200'); + }, + ); + }); }); describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { @@ -844,6 +1070,7 @@ describe('NetworkController', () => { autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); + createNetworkClientMock.mockReturnValue(buildFakeClient()); controller.disableRpcFailover(); @@ -860,6 +1087,230 @@ describe('NetworkController', () => { }, ); }); + + it('destroys only the existing network clients whose RPC endpoints have configured failover URLs', async () => { + await withController( + { + isRpcFailoverEnabled: true, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + [ChainId.mainnet]: buildInfuraNetworkConfiguration( + InfuraNetworkType.mainnet, + { + rpcEndpoints: [ + buildInfuraRpcEndpoint(InfuraNetworkType.mainnet, { + failoverUrls: [], + }), + ], + }, + ), + '0x200': buildCustomNetworkConfiguration({ + chainId: '0x200', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network/1', + failoverUrls: ['https://failover.endpoint/1'], + }), + ], + }), + '0x300': buildCustomNetworkConfiguration({ + chainId: '0x300', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'BBBB-BBBB-BBBB-BBBB', + url: 'https://test.network/2', + failoverUrls: ['https://failover.endpoint/2'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const originalCreateAutoManagedNetworkClient = + createAutoManagedNetworkClientModule.createAutoManagedNetworkClient; + const autoManagedNetworkClients: AutoManagedNetworkClient[] = + []; + jest + .spyOn( + createAutoManagedNetworkClientModule, + 'createAutoManagedNetworkClient', + ) + .mockImplementation((...args) => { + const autoManagedNetworkClient = + originalCreateAutoManagedNetworkClient(...args); + jest.spyOn(autoManagedNetworkClient, 'destroy'); + autoManagedNetworkClients.push(autoManagedNetworkClient); + return autoManagedNetworkClient; + }); + createNetworkClientMock.mockReturnValue(buildFakeClient()); + + controller.disableRpcFailover(); + await flushPromises(); + + expect(autoManagedNetworkClients).toHaveLength(3); + expect(autoManagedNetworkClients[0].destroy).not.toHaveBeenCalled(); + expect(autoManagedNetworkClients[1].destroy).toHaveBeenCalled(); + expect(autoManagedNetworkClients[2].destroy).toHaveBeenCalled(); + }, + ); + }); + + it('updates the provider proxy to point to the new selected network client if it was recreated', async () => { + await withController( + { + isRpcFailoverEnabled: true, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + '0x100': buildCustomNetworkConfiguration({ + chainId: '0x100', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network', + failoverUrls: ['https://failover.endpoint'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const fakeProviders = [ + buildFakeProvider([ + { + request: { + method: 'test_method', + }, + response: { + result: 'response from provider 1', + }, + }, + ]), + buildFakeProvider([ + { + request: { + method: 'test_method', + }, + response: { + result: 'response from provider 2', + }, + }, + ]), + ]; + const fakeNetworkClients = [ + buildFakeClient(fakeProviders[0]), + buildFakeClient(fakeProviders[1]), + ]; + let i = 0; + createNetworkClientMock.mockImplementation(({ configuration }) => { + if (i === 0) { + i += 1; + return fakeNetworkClients[0]; + } else if (i === 1) { + i += 1; + return fakeNetworkClients[1]; + } + throw new Error( + `Unknown network client configuration ${JSON.stringify( + configuration, + )}`, + ); + }); + await controller.initializeProvider(); + + controller.disableRpcFailover(); + await flushPromises(); + + const { provider } = controller.getProviderAndBlockTracker(); + assert(provider, 'Provider is somehow unset'); + const result = await provider.request({ + id: '1', + jsonrpc: '2.0', + method: 'test_method', + }); + expect(result).toBe('response from provider 2'); + }, + ); + }); + + it('updates the block tracker proxy to point to the new selected network client if it was recreated', async () => { + await withController( + { + isRpcFailoverEnabled: true, + state: { + selectedNetworkClientId: 'AAAA-AAAA-AAAA-AAAA', + networkConfigurationsByChainId: { + '0x100': buildCustomNetworkConfiguration({ + chainId: '0x100', + rpcEndpoints: [ + buildCustomRpcEndpoint({ + networkClientId: 'AAAA-AAAA-AAAA-AAAA', + url: 'https://test.network', + failoverUrls: ['https://failover.endpoint'], + }), + ], + }), + }, + }, + }, + async ({ controller }) => { + const fakeProviders = [ + buildFakeProvider([ + { + request: { + method: 'eth_blockNumber', + }, + response: { + result: '0x100', + }, + }, + ]), + buildFakeProvider([ + { + request: { + method: 'eth_blockNumber', + }, + response: { + result: '0x200', + }, + }, + ]), + ]; + const fakeNetworkClients = [ + buildFakeClient(fakeProviders[0]), + buildFakeClient(fakeProviders[1]), + ]; + let i = 0; + createNetworkClientMock.mockImplementation(({ configuration }) => { + if (i === 0) { + i += 1; + return fakeNetworkClients[0]; + } else if (i === 1) { + i += 1; + return fakeNetworkClients[1]; + } + throw new Error( + `Unknown network client configuration ${JSON.stringify( + configuration, + )}`, + ); + }); + await controller.initializeProvider(); + + controller.disableRpcFailover(); + await flushPromises(); + + const { blockTracker } = controller.getProviderAndBlockTracker(); + assert(blockTracker, 'Block tracker is somehow unset'); + const latestBlockNumber = await blockTracker.getLatestBlock(); + expect(latestBlockNumber).toBe('0x200'); + }, + ); + }); }); describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { @@ -920,6 +1371,7 @@ describe('NetworkController', () => { autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); + createNetworkClientMock.mockReturnValue(buildFakeClient()); controller.disableRpcFailover(); @@ -15234,7 +15686,7 @@ function buildFakeClient( rpcUrl: 'https://test.network', }, provider, - blockTracker: new FakeBlockTracker({ provider }), + blockTracker: new PollingBlockTracker({ provider }), destroy: () => { // do nothing }, diff --git a/packages/network-controller/tests/helpers.ts b/packages/network-controller/tests/helpers.ts index 30def58c9ef..81627d3154d 100644 --- a/packages/network-controller/tests/helpers.ts +++ b/packages/network-controller/tests/helpers.ts @@ -102,7 +102,7 @@ export function buildNetworkControllerMessenger( * requests. * @returns The fake network client. */ -function buildFakeNetworkClient({ +export function buildFakeNetworkClient({ configuration, providerStubs = [], }: { From a898873fb54a1cf4c1ffe2d6013de85d03f04200 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Sun, 20 Apr 2025 23:11:53 -0600 Subject: [PATCH 15/17] Fix tests --- .../src/NetworkController.ts | 7 +- ...create-auto-managed-network-client.test.ts | 94 +++++++++++++++++++ .../src/create-auto-managed-network-client.ts | 4 +- .../tests/NetworkController.test.ts | 2 +- 4 files changed, 99 insertions(+), 8 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index 641a5bd37e6..711acf018e2 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -1093,10 +1093,7 @@ export class NetworkController extends BaseController< readonly #getRpcServiceOptions: NetworkControllerOptions['getRpcServiceOptions']; - readonly #getBlockTrackerOptions: Exclude< - NetworkControllerOptions['getBlockTrackerOptions'], - undefined - >; + readonly #getBlockTrackerOptions: NetworkControllerOptions['getBlockTrackerOptions']; #networkConfigurationsByNetworkClientId: Map< NetworkClientId, @@ -1120,7 +1117,7 @@ export class NetworkController extends BaseController< infuraProjectId, log, getRpcServiceOptions, - getBlockTrackerOptions = () => ({}), + getBlockTrackerOptions, additionalDefaultNetworks, isRpcFailoverEnabled = false, } = options; diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index 6eba2e69247..c55d56f5563 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -197,6 +197,55 @@ describe('createAutoManagedNetworkClient', () => { }); }); + it('allows block tracker options to be optional', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'test_method', + params: [], + }, + response: { + result: 'test response', + }, + discardAfterMatching: false, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const { provider } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + expect(createNetworkClientMock).toHaveBeenCalledTimes(1); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + getBlockTrackerOptions: expect.any(Function), + messenger, + isRpcFailoverEnabled: true, + }); + }); + it('allows for enabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, @@ -461,6 +510,51 @@ describe('createAutoManagedNetworkClient', () => { }); }); + it('allows block tracker options to be optional', async () => { + mockNetwork({ + networkClientConfiguration, + mocks: [ + { + request: { + method: 'eth_blockNumber', + params: [], + }, + response: { + result: '0x1', + }, + }, + ], + }); + const createNetworkClientMock = jest.spyOn( + createNetworkClientModule, + 'createNetworkClient', + ); + const getRpcServiceOptions = () => ({ + btoa, + fetch, + }); + const messenger = getNetworkControllerMessenger(); + + const { blockTracker } = createAutoManagedNetworkClient({ + networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + expect(createNetworkClientMock).toHaveBeenCalledTimes(1); + expect(createNetworkClientMock).toHaveBeenCalledWith({ + configuration: networkClientConfiguration, + getRpcServiceOptions, + getBlockTrackerOptions: expect.any(Function), + messenger, + isRpcFailoverEnabled: true, + }); + }); + it('allows for enabling the RPC failover behavior', async () => { mockNetwork({ networkClientConfiguration, diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index 1897d69e610..499b5234b71 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -83,7 +83,7 @@ export function createAutoManagedNetworkClient< >({ networkClientConfiguration, getRpcServiceOptions, - getBlockTrackerOptions, + getBlockTrackerOptions = () => ({}), messenger, isRpcFailoverEnabled, }: { @@ -91,7 +91,7 @@ export function createAutoManagedNetworkClient< getRpcServiceOptions: ( rpcEndpointUrl: string, ) => Omit; - getBlockTrackerOptions: ( + getBlockTrackerOptions?: ( rpcEndpointUrl: string, ) => Omit; messenger: NetworkControllerMessenger; diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index a1ee46a68c0..214f210bd2d 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -40,6 +40,7 @@ import { } from './helpers'; import type { FakeProviderStub } from '../../../tests/fake-provider'; import { FakeProvider } from '../../../tests/fake-provider'; +import { flushPromises } from '../../../tests/helpers'; import { NetworkStatus } from '../src/constants'; import * as createAutoManagedNetworkClientModule from '../src/create-auto-managed-network-client'; import type { AutoManagedNetworkClient } from '../src/create-auto-managed-network-client'; @@ -69,7 +70,6 @@ import { } from '../src/NetworkController'; import type { NetworkClientConfiguration, Provider } from '../src/types'; import { NetworkClientType } from '../src/types'; -import { flushPromises } from '../../../tests/helpers'; jest.mock('../src/create-network-client'); From b0a8055bb636d9a02a6d28e14a4f6dd05cee9e23 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Tue, 22 Apr 2025 14:40:33 -0600 Subject: [PATCH 16/17] Fix --- .../src/NetworkController.ts | 11 +-- ...create-auto-managed-network-client.test.ts | 76 ++++++++++++++++--- .../src/create-auto-managed-network-client.ts | 30 +++----- .../tests/NetworkController.test.ts | 40 ++++++---- .../block-hash-in-response.ts | 2 +- 5 files changed, 106 insertions(+), 53 deletions(-) diff --git a/packages/network-controller/src/NetworkController.ts b/packages/network-controller/src/NetworkController.ts index 26d2ceff1bd..c8b8c4ca68d 100644 --- a/packages/network-controller/src/NetworkController.ts +++ b/packages/network-controller/src/NetworkController.ts @@ -52,7 +52,6 @@ import type { NetworkClientConfiguration, AdditionalDefaultNetwork, } from './types'; -import { NetworkClient } from './create-network-client'; const debugLog = createModuleLogger(projectLogger, 'NetworkController'); @@ -1290,13 +1289,9 @@ export class NetworkController extends BaseController< networkClient.configuration.failoverRpcUrls && networkClient.configuration.failoverRpcUrls.length > 0 ) { - networkClient.destroy(); - const newNetworkClient = newIsRpcFailoverEnabled - ? networkClient.withRpcFailoverEnabled() - : networkClient.withRpcFailoverDisabled(); - networkClientsById[ - networkClientId as keyof typeof networkClientsById - ] = newNetworkClient; + newIsRpcFailoverEnabled + ? networkClient.enableRpcFailover() + : networkClient.disableRpcFailover(); } } } diff --git a/packages/network-controller/src/create-auto-managed-network-client.test.ts b/packages/network-controller/src/create-auto-managed-network-client.test.ts index 01f00cad5f8..e604c403d3a 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.test.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.test.ts @@ -188,7 +188,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover behavior', async () => { + it('allows for enabling the RPC failover behavior, even after having already accessed the provider', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -219,7 +219,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: false, - }).withRpcFailoverEnabled(); + }); const { provider } = autoManagedNetworkClient; await provider.request({ @@ -228,7 +228,21 @@ describe('createAutoManagedNetworkClient', () => { method: 'test_method', params: [], }); - expect(createNetworkClientMock).toHaveBeenCalledWith({ + autoManagedNetworkClient.enableRpcFailover(); + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + + expect(createNetworkClientMock).toHaveBeenNthCalledWith(1, { + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); + expect(createNetworkClientMock).toHaveBeenNthCalledWith(2, { configuration: networkClientConfiguration, getRpcServiceOptions, messenger, @@ -236,7 +250,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for disabling the RPC failover behavior', async () => { + it('allows for disabling the RPC failover behavior, even after having accessed the provider', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -267,7 +281,7 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: true, - }).withRpcFailoverDisabled(); + }); const { provider } = autoManagedNetworkClient; await provider.request({ @@ -276,7 +290,21 @@ describe('createAutoManagedNetworkClient', () => { method: 'test_method', params: [], }); - expect(createNetworkClientMock).toHaveBeenCalledWith({ + autoManagedNetworkClient.disableRpcFailover(); + await provider.request({ + id: 1, + jsonrpc: '2.0', + method: 'test_method', + params: [], + }); + + expect(createNetworkClientMock).toHaveBeenNthCalledWith(1, { + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + expect(createNetworkClientMock).toHaveBeenNthCalledWith(2, { configuration: networkClientConfiguration, getRpcServiceOptions, messenger, @@ -435,7 +463,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for enabling the RPC failover behavior', async () => { + it('allows for enabling the RPC failover behavior, even after having already accessed the provider', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -447,6 +475,7 @@ describe('createAutoManagedNetworkClient', () => { response: { result: '0x1', }, + discardAfterMatching: false, }, ], }); @@ -465,13 +494,24 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: false, - }).withRpcFailoverEnabled(); + }); const { blockTracker } = autoManagedNetworkClient; await new Promise((resolve) => { blockTracker.once('latest', resolve); }); - expect(createNetworkClientMock).toHaveBeenCalledWith({ + autoManagedNetworkClient.enableRpcFailover(); + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + + expect(createNetworkClientMock).toHaveBeenNthCalledWith(1, { + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: false, + }); + expect(createNetworkClientMock).toHaveBeenNthCalledWith(2, { configuration: networkClientConfiguration, getRpcServiceOptions, messenger, @@ -479,7 +519,7 @@ describe('createAutoManagedNetworkClient', () => { }); }); - it('allows for disabling the RPC failover behavior', async () => { + it('allows for disabling the RPC failover behavior, even after having already accessed the provider', async () => { mockNetwork({ networkClientConfiguration, mocks: [ @@ -491,6 +531,7 @@ describe('createAutoManagedNetworkClient', () => { response: { result: '0x1', }, + discardAfterMatching: false, }, ], }); @@ -509,13 +550,24 @@ describe('createAutoManagedNetworkClient', () => { getRpcServiceOptions, messenger, isRpcFailoverEnabled: true, - }).withRpcFailoverDisabled(); + }); const { blockTracker } = autoManagedNetworkClient; await new Promise((resolve) => { blockTracker.once('latest', resolve); }); - expect(createNetworkClientMock).toHaveBeenCalledWith({ + autoManagedNetworkClient.disableRpcFailover(); + await new Promise((resolve) => { + blockTracker.once('latest', resolve); + }); + + expect(createNetworkClientMock).toHaveBeenNthCalledWith(1, { + configuration: networkClientConfiguration, + getRpcServiceOptions, + messenger, + isRpcFailoverEnabled: true, + }); + expect(createNetworkClientMock).toHaveBeenNthCalledWith(2, { configuration: networkClientConfiguration, getRpcServiceOptions, messenger, diff --git a/packages/network-controller/src/create-auto-managed-network-client.ts b/packages/network-controller/src/create-auto-managed-network-client.ts index d18cc102993..9c3264f4070 100644 --- a/packages/network-controller/src/create-auto-managed-network-client.ts +++ b/packages/network-controller/src/create-auto-managed-network-client.ts @@ -40,8 +40,8 @@ export type AutoManagedNetworkClient< provider: ProxyWithAccessibleTarget; blockTracker: ProxyWithAccessibleTarget; destroy: () => void; - withRpcFailoverEnabled: () => AutoManagedNetworkClient; - withRpcFailoverDisabled: () => AutoManagedNetworkClient; + enableRpcFailover: () => void; + disableRpcFailover: () => void; }; /** @@ -203,22 +203,16 @@ export function createAutoManagedNetworkClient< networkClient?.destroy(); }; - const withRpcFailoverEnabled = () => { - return createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions, - messenger, - isRpcFailoverEnabled: true, - }); + const enableRpcFailover = () => { + isRpcFailoverEnabled = true; + destroy(); + networkClient = undefined; }; - const withRpcFailoverDisabled = () => { - return createAutoManagedNetworkClient({ - networkClientConfiguration, - getRpcServiceOptions, - messenger, - isRpcFailoverEnabled: false, - }); + const disableRpcFailover = () => { + isRpcFailoverEnabled = false; + destroy(); + networkClient = undefined; }; return { @@ -226,7 +220,7 @@ export function createAutoManagedNetworkClient< provider: providerProxy, blockTracker: blockTrackerProxy, destroy, - withRpcFailoverEnabled, - withRpcFailoverDisabled, + enableRpcFailover, + disableRpcFailover, }; } diff --git a/packages/network-controller/tests/NetworkController.test.ts b/packages/network-controller/tests/NetworkController.test.ts index 982fccfda75..990698f18c6 100644 --- a/packages/network-controller/tests/NetworkController.test.ts +++ b/packages/network-controller/tests/NetworkController.test.ts @@ -642,7 +642,7 @@ describe('NetworkController', () => { describe('enableRpcFailover', () => { describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { - it.only('calls withRpcFailoverEnabled on only the network clients whose RPC endpoints have configured failover URLs', async () => { + it('calls enableRpcFailover on only the network clients whose RPC endpoints have configured failover URLs', async () => { await withController( { isRpcFailoverEnabled: false, @@ -695,7 +695,7 @@ describe('NetworkController', () => { .mockImplementation((...args) => { const autoManagedNetworkClient = originalCreateAutoManagedNetworkClient(...args); - jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverEnabled'); + jest.spyOn(autoManagedNetworkClient, 'enableRpcFailover'); autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); @@ -704,13 +704,13 @@ describe('NetworkController', () => { expect(autoManagedNetworkClients).toHaveLength(3); expect( - autoManagedNetworkClients[0].withRpcFailoverEnabled, + autoManagedNetworkClients[0].enableRpcFailover, ).not.toHaveBeenCalled(); expect( - autoManagedNetworkClients[1].withRpcFailoverEnabled, + autoManagedNetworkClients[1].enableRpcFailover, ).toHaveBeenCalled(); expect( - autoManagedNetworkClients[2].withRpcFailoverEnabled, + autoManagedNetworkClients[2].enableRpcFailover, ).toHaveBeenCalled(); }, ); @@ -718,7 +718,7 @@ describe('NetworkController', () => { }); describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { - it.only('does not call createAutoManagedNetworkClient at all', async () => { + it('does not call createAutoManagedNetworkClient at all', async () => { await withController( { isRpcFailoverEnabled: true, @@ -771,7 +771,7 @@ describe('NetworkController', () => { .mockImplementation((...args) => { const autoManagedNetworkClient = originalCreateAutoManagedNetworkClient(...args); - jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverEnabled'); + jest.spyOn(autoManagedNetworkClient, 'enableRpcFailover'); autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); @@ -787,7 +787,7 @@ describe('NetworkController', () => { describe('disableRpcFailover', () => { describe('if the controller was initialized with isRpcFailoverEnabled = true', () => { - it.only('calls withRpcFailoverDisabled on only the network clients whose RPC endpoints have configured failover URLs', async () => { + it('calls disableRpcFailover on only the network clients whose RPC endpoints have configured failover URLs', async () => { await withController( { isRpcFailoverEnabled: true, @@ -840,7 +840,7 @@ describe('NetworkController', () => { .mockImplementation((...args) => { const autoManagedNetworkClient = originalCreateAutoManagedNetworkClient(...args); - jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverDisabled'); + jest.spyOn(autoManagedNetworkClient, 'disableRpcFailover'); autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); @@ -849,13 +849,13 @@ describe('NetworkController', () => { expect(autoManagedNetworkClients).toHaveLength(3); expect( - autoManagedNetworkClients[0].withRpcFailoverDisabled, + autoManagedNetworkClients[0].disableRpcFailover, ).not.toHaveBeenCalled(); expect( - autoManagedNetworkClients[1].withRpcFailoverDisabled, + autoManagedNetworkClients[1].disableRpcFailover, ).toHaveBeenCalled(); expect( - autoManagedNetworkClients[2].withRpcFailoverDisabled, + autoManagedNetworkClients[2].disableRpcFailover, ).toHaveBeenCalled(); }, ); @@ -863,7 +863,7 @@ describe('NetworkController', () => { }); describe('if the controller was initialized with isRpcFailoverEnabled = false', () => { - it.only('does not call createAutoManagedNetworkClient at all', async () => { + it('does not call createAutoManagedNetworkClient at all', async () => { await withController( { isRpcFailoverEnabled: false, @@ -916,7 +916,7 @@ describe('NetworkController', () => { .mockImplementation((...args) => { const autoManagedNetworkClient = originalCreateAutoManagedNetworkClient(...args); - jest.spyOn(autoManagedNetworkClient, 'withRpcFailoverDisabled'); + jest.spyOn(autoManagedNetworkClient, 'disableRpcFailover'); autoManagedNetworkClients.push(autoManagedNetworkClient); return autoManagedNetworkClient; }); @@ -1522,6 +1522,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, 'linea-sepolia': { blockTracker: expect.anything(), @@ -1535,6 +1537,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, mainnet: { blockTracker: expect.anything(), @@ -1548,6 +1552,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, sepolia: { blockTracker: expect.anything(), @@ -1561,6 +1567,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, }); }, @@ -1615,6 +1623,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, 'BBBB-BBBB-BBBB-BBBB': { blockTracker: expect.anything(), @@ -1627,6 +1637,8 @@ describe('NetworkController', () => { }, provider: expect.anything(), destroy: expect.any(Function), + enableRpcFailover: expect.any(Function), + disableRpcFailover: expect.any(Function), }, }); }, diff --git a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts index 576896ace64..95fc8c1f68b 100644 --- a/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts +++ b/packages/network-controller/tests/provider-api-tests/block-hash-in-response.ts @@ -6,8 +6,8 @@ import { withMockedCommunications, withNetworkClient, } from './helpers'; -import { NetworkClientType } from '../../src/types'; import { testsForRpcFailoverBehavior } from './rpc-failover'; +import { NetworkClientType } from '../../src/types'; type TestsForRpcMethodThatCheckForBlockHashInResponseOptions = { providerType: ProviderType; From 96398895914f49b942aa412ef8a7d8822e499b24 Mon Sep 17 00:00:00 2001 From: Elliot Winkler Date: Tue, 22 Apr 2025 14:48:05 -0600 Subject: [PATCH 17/17] Fix yarn.lock --- yarn.lock | 3 --- 1 file changed, 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index a91abd5cd24..76ce1189795 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3818,7 +3818,6 @@ __metadata: "@metamask/eth-json-rpc-provider": "npm:^4.1.8" "@metamask/eth-query": "npm:^4.0.0" "@metamask/json-rpc-engine": "npm:^10.0.3" - "@metamask/remote-feature-flag-controller": "npm:^1.6.0" "@metamask/rpc-errors": "npm:^7.0.2" "@metamask/swappable-obj-proxy": "npm:^2.3.0" "@metamask/utils": "npm:^11.2.0" @@ -3844,8 +3843,6 @@ __metadata: typescript: "npm:~5.2.2" uri-js: "npm:^4.4.1" uuid: "npm:^8.3.2" - peerDependencies: - "@metamask/remote-feature-flag-controller": ^1.5.0 languageName: unknown linkType: soft