From 1990b2b511361bca2347bb56f09e354193dcf47f Mon Sep 17 00:00:00 2001 From: Lukas Date: Fri, 12 Jan 2024 10:58:35 +0100 Subject: [PATCH] fix: log error (#5) * fix: log error * fix: add client map * fix: quicknode is slow * fix: improve getLogs * fix: remove snapshot --- src/rpc/__snapshots__/helpers.spec.ts.snap | 2 +- src/rpc/chainIds.ts | 1 - src/rpc/clients.ts | 50 ++++++++---- src/rpc/helpers.spec.ts | 12 +-- src/rpc/helpers.ts | 94 +++++++++++++--------- 5 files changed, 99 insertions(+), 60 deletions(-) diff --git a/src/rpc/__snapshots__/helpers.spec.ts.snap b/src/rpc/__snapshots__/helpers.spec.ts.snap index 7f70be4..d3f5174 100644 --- a/src/rpc/__snapshots__/helpers.spec.ts.snap +++ b/src/rpc/__snapshots__/helpers.spec.ts.snap @@ -1,6 +1,6 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`helpers > getPastLogsRecursive 1`] = ` +exports[`helpers > getLogs should use batching for known rpcs 1`] = ` [ { "address": "0x398ec7346dcd622edc5ae82352f02be94c62d119", diff --git a/src/rpc/chainIds.ts b/src/rpc/chainIds.ts index 7b340ac..b1638f6 100644 --- a/src/rpc/chainIds.ts +++ b/src/rpc/chainIds.ts @@ -3,7 +3,6 @@ export enum ChainId { ropsten = 3, rinkeby = 4, goerli = 5, - xdai = 100, polygon = 137, mumbai = 80001, avalanche = 43114, diff --git a/src/rpc/clients.ts b/src/rpc/clients.ts index 2bd588e..c319f3b 100644 --- a/src/rpc/clients.ts +++ b/src/rpc/clients.ts @@ -1,4 +1,4 @@ -import { createPublicClient, http } from 'viem'; +import { PublicClient, createPublicClient, http } from 'viem'; import { mainnet, arbitrum, @@ -15,73 +15,93 @@ import { polygonZkEvm, scroll, } from 'viem/chains'; +import { ChainId } from './chainIds'; + +const commonConfig = { timeout: 30_000 }; export const mainnetClient = createPublicClient({ chain: mainnet, - transport: http('https://eth.llamarpc.com'), //process.env.RPC_MAINNET), + transport: http(process.env.RPC_MAINNET, commonConfig), }); export const arbitrumClient = createPublicClient({ chain: arbitrum, - transport: http(process.env.RPC_ARBITRUM), + transport: http(process.env.RPC_ARBITRUM, commonConfig), }); export const polygonClient = createPublicClient({ chain: polygon, - transport: http(process.env.RPC_POLYGON), + transport: http(process.env.RPC_POLYGON, commonConfig), }); export const optimismClient = createPublicClient({ chain: optimism, - transport: http(process.env.RPC_OPTIMISM), + transport: http(process.env.RPC_OPTIMISM, commonConfig), }); export const metisClient = createPublicClient({ chain: metis, - transport: http(process.env.RPC_METIS), + transport: http(process.env.RPC_METIS, commonConfig), }); export const baseClient = createPublicClient({ chain: base, - transport: http(process.env.RPC_BASE), + transport: http(process.env.RPC_BASE, commonConfig), }); export const fantomClient = createPublicClient({ chain: fantom, - transport: http(process.env.RPC_FANTOM), + transport: http(process.env.RPC_FANTOM, commonConfig), }); export const bnbClient = createPublicClient({ chain: bsc, - transport: http(process.env.RPC_BNB), + transport: http(process.env.RPC_BNB, commonConfig), }); export const avalancheClient = createPublicClient({ chain: avalanche, - transport: http(process.env.RPC_AVALANCHE), + transport: http(process.env.RPC_AVALANCHE, commonConfig), }); export const gnosisClient = createPublicClient({ chain: gnosis, - transport: http(process.env.RPC_GNOSIS), + transport: http(process.env.RPC_GNOSIS, commonConfig), }); export const sepoliaClient = createPublicClient({ chain: sepolia, - transport: http(process.env.RPC_SEPOLIA), + transport: http(process.env.RPC_SEPOLIA, commonConfig), }); export const goerliClient = createPublicClient({ chain: goerli, - transport: http(process.env.RPC_GOERLI), + transport: http(process.env.RPC_GOERLI, commonConfig), }); export const scrollClient = createPublicClient({ chain: scroll, - transport: http(process.env.RPC_SCROLL), + transport: http(process.env.RPC_SCROLL, commonConfig), }); export const zkEVMClient = createPublicClient({ chain: polygonZkEvm, - transport: http(process.env.RPC_ZKEVM), + transport: http(process.env.RPC_ZKEVM, commonConfig), }); + +export const CHAIN_ID_CLIENT_MAP: Record = { + [ChainId.mainnet]: mainnetClient, + [ChainId.arbitrum_one]: arbitrumClient, + [ChainId.polygon]: polygonClient, + [ChainId.optimism]: optimismClient, + [ChainId.metis]: metisClient, + [ChainId.base]: baseClient, + [ChainId.sepolia]: sepoliaClient, + [ChainId.goerli]: goerliClient, + [ChainId.fantom]: fantomClient, + [ChainId.bnb]: bnbClient, + [ChainId.avalanche]: avalancheClient, + [ChainId.gnosis]: gnosisClient, + [ChainId.scroll]: scrollClient, + [ChainId.zkEVM]: zkEVMClient, +} as const; diff --git a/src/rpc/helpers.spec.ts b/src/rpc/helpers.spec.ts index 46465fd..56de3f8 100644 --- a/src/rpc/helpers.spec.ts +++ b/src/rpc/helpers.spec.ts @@ -1,12 +1,14 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { getContractDeploymentBlock, getBlockAtTimestamp, getLogsRecursive, + getLogs, } from './helpers'; import { mainnetClient } from './clients'; -import { getAbiItem } from 'viem'; +import { createPublicClient, fallback, getAbiItem, http } from 'viem'; import { IPoolV1_ABI } from './mocks/IPoolV1'; +import { mainnet } from 'viem/chains'; describe('helpers', () => { it( @@ -42,16 +44,16 @@ describe('helpers', () => { }); it( - 'getPastLogsRecursive', + 'getLogs should use batching for known rpcs', async () => { - const logs = await getLogsRecursive({ + const logs = await getLogs({ client: mainnetClient, events: [getAbiItem({ abi: IPoolV1_ABI, name: 'Borrow' })], address: '0x398eC7346DcD622eDc5ae82352F02bE94C62d119', // v1 pool fromBlock: 9241022n, toBlock: 9281022n, }); - + expect(logs.length).gt(0); expect(logs).toMatchSnapshot(); }, { timeout: 30000 }, diff --git a/src/rpc/helpers.ts b/src/rpc/helpers.ts index 096d056..ae92233 100644 --- a/src/rpc/helpers.ts +++ b/src/rpc/helpers.ts @@ -1,4 +1,4 @@ -import { Address, GetLogsReturnType, PublicClient } from 'viem'; +import { Address, GetLogsReturnType, PublicClient, Transport } from 'viem'; import type { Abi, AbiEvent } from 'abitype'; interface GetContractDeploymentBlockArgs { @@ -110,7 +110,7 @@ export async function getBlockAtTimestamp({ throw new Error('Could not find matching block'); } -interface GetLogsRecursiveArgs { +interface GetLogsArgs { client: PublicClient; events: TAbiEvents; address: Address; @@ -118,62 +118,80 @@ interface GetLogsRecursiveArgs { toBlock: bigint; } -/** - * fetches logs recursively - */ -export async function getLogsRecursive< - TAbiEvents extends AbiEvent[] | undefined, ->({ +export async function getLogs({ client, events, address, fromBlock, toBlock, -}: GetLogsRecursiveArgs): Promise< - GetLogsReturnType -> { - if (fromBlock <= toBlock) { - try { - const logs = await client.getLogs({ +}: GetLogsArgs): Promise> { + if (client.transport.key === 'http') { + const url: string = client.transport.url; + console.log(url); + if (/llamarpc/.test(url)) + return getLogsInBatches({ + client, + events, + address, fromBlock, toBlock, + batchSize: 100_000, + }); + if (/quiknode/.test(url)) + return getLogsInBatches({ + client, events, address, + fromBlock, + toBlock, + batchSize: 10_000, }); - return logs; - } catch (error: any) { - // quicknode style errors - if ( - error.message && - (error.message as string).includes( - 'eth_getLogs is limited to a 10,000 range', - ) - ) { - return getLogsInBatches({ - client, - events, - address, + // alchemy behaves different to other rpcs as it allows querying with infinite block range as long as the response size is below a certain threshold + if (/alchemy/.test(url)) { + try { + return await client.getLogs({ fromBlock, toBlock, - batchSize: 10000, + events, + address, }); - } - // llama style error - if ( - error.message && - (error.message as string).includes( - 'query exceeds max block range 100000', - ) - ) { + } catch (e) { return getLogsInBatches({ client, events, address, fromBlock, toBlock, - batchSize: 100000, + batchSize: 2_000, }); } + } + } + return getLogsRecursive({ client, events, address, fromBlock, toBlock }); +} + +/** + * fetches logs recursively + */ +export async function getLogsRecursive< + TAbiEvents extends AbiEvent[] | undefined, +>({ + client, + events, + address, + fromBlock, + toBlock, +}: GetLogsArgs): Promise> { + if (fromBlock <= toBlock) { + try { + const logs = await client.getLogs({ + fromBlock, + toBlock, + events, + address, + }); + return logs; + } catch (error: any) { // divide & conquer when issue/limit is now known const midBlock = BigInt(fromBlock + toBlock) >> BigInt(1); const arr1 = await getLogsRecursive({ @@ -197,7 +215,7 @@ export async function getLogsRecursive< } interface GetLogsInBatchesArgs - extends GetLogsRecursiveArgs { + extends GetLogsArgs { batchSize: number; }