From 5098d34ed00008b9ea39a32ce6da99cf6e4d1ef5 Mon Sep 17 00:00:00 2001 From: Alonso Villegas Date: Fri, 1 Nov 2024 12:38:05 +0100 Subject: [PATCH] batch nile user positions fetching --- adapters/nile/hourly_blocks.csv | 2 +- adapters/nile/src/index.ts | 81 +++++++++------------ adapters/nile/src/sdk/config.ts | 11 +-- adapters/nile/src/sdk/lensDetails.ts | 103 ++++++++++++++++----------- adapters/nile/src/sdk/lockers.ts | 26 +++++++ 5 files changed, 130 insertions(+), 93 deletions(-) create mode 100644 adapters/nile/src/sdk/lockers.ts diff --git a/adapters/nile/hourly_blocks.csv b/adapters/nile/hourly_blocks.csv index d862ab16..76ee32f8 100644 --- a/adapters/nile/hourly_blocks.csv +++ b/adapters/nile/hourly_blocks.csv @@ -1,2 +1,2 @@ number,timestamp -3976979,1713974398 +11449898,1730365248 diff --git a/adapters/nile/src/index.ts b/adapters/nile/src/index.ts index c1348ac9..166cf580 100644 --- a/adapters/nile/src/index.ts +++ b/adapters/nile/src/index.ts @@ -1,6 +1,6 @@ import fs from "fs"; import { write } from "fast-csv"; -import csv from 'csv-parser'; +import csv from "csv-parser"; import { BlockData, OutputSchemaRow, UserVote } from "./sdk/types"; import { getV2UserPositionsAtBlock, @@ -9,6 +9,7 @@ import { import { getTimestampAtBlock } from "./sdk/common"; import { fetchUserVotes } from "./sdk/lensDetails"; import BigNumber from "bignumber.js"; +import { getUserLockers } from "./sdk/lockers"; const NILE_ADDRESS = "0xAAAac83751090C6ea42379626435f805DDF54DC8".toLowerCase(); @@ -82,14 +83,8 @@ export const getUserLiquidityTVLByBlock = async ({ getV3UserPositionsAtBlock(blockNumber), ]); - const userAddresses = [...v2Positions, ...v3Positions] - .map((pos) => pos.user) - .reduce( - (prev, curr) => (prev.includes(curr) ? prev : [...prev, curr]), - [] as string[], - ); - - const userVotes = await getUserVotesTVLByBlock(blockNumber, userAddresses); + const userLockers = await getUserLockers(blockNumber); + const userVotes = await getUserVotesTVLByBlock(blockNumber, userLockers); // combine v2 & v3 const combinedPositions = [...v2Positions, ...v3Positions]; @@ -142,24 +137,16 @@ export const getUserVotesTVLByBlock = async ( [userAddress: string]: BigNumber; }; - const batchSize = 300; - let userVotesResult: any[] = []; - for (let i = 0; i < userAddresses.length; i += batchSize) { - const batch = userAddresses.slice(i, i + batchSize); - userVotesResult = userVotesResult.concat( - await Promise.all( - batch.map((user) => fetchUserVotes(BigInt(blockNumber), user)), - ), - ); - } + const userVotesResult = await fetchUserVotes( + BigInt(blockNumber), + userAddresses, + ); for (const userFecthedVotes of userVotesResult) { - for (const userVote of userFecthedVotes) { - const userAddress = userVote.result.userAddress.toLowerCase(); - tokenBalanceMap[userAddress] = BigNumber( - tokenBalanceMap[userAddress] ?? 0, - ).plus(userVote.result.amount.toString()); - } + const userAddress = userFecthedVotes.result.userAddress.toLowerCase(); + tokenBalanceMap[userAddress] = BigNumber( + tokenBalanceMap[userAddress] ?? 0, + ).plus(userFecthedVotes.result.amount.toString()); } Object.entries(tokenBalanceMap).forEach(([userAddress, balance]) => { @@ -178,24 +165,23 @@ export const getUserVotesTVLByBlock = async ( // console.log("Done"); // }); - const readBlocksFromCSV = async (filePath: string): Promise => { const blocks: BlockData[] = []; await new Promise((resolve, reject) => { fs.createReadStream(filePath) .pipe(csv()) // Specify the separator as '\t' for TSV files - .on('data', (row) => { + .on("data", (row) => { const blockNumber = parseInt(row.number, 10); const blockTimestamp = parseInt(row.timestamp, 10); if (!isNaN(blockNumber) && blockTimestamp) { blocks.push({ blockNumber: blockNumber, blockTimestamp }); } }) - .on('end', () => { + .on("end", () => { resolve(); }) - .on('error', (err) => { + .on("error", (err) => { reject(err); }); }); @@ -203,29 +189,30 @@ const readBlocksFromCSV = async (filePath: string): Promise => { return blocks; }; +readBlocksFromCSV("hourly_blocks.csv") + .then(async (blocks: any[]) => { + console.log(blocks); + const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks -readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => { - console.log(blocks); - const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks - - for (const block of blocks) { + for (const block of blocks) { try { - const result = await getUserTVLByBlock(block); - // Accumulate CSV rows for all blocks - allCsvRows.push(...result); + const result = await getUserTVLByBlock(block); + // Accumulate CSV rows for all blocks + allCsvRows.push(...result); } catch (error) { - console.error(`An error occurred for block ${block}:`, error); + console.error(`An error occurred for block ${block}:`, error); } - } - await new Promise((resolve, reject) => { - const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' }); - write(allCsvRows, { headers: true }) + } + await new Promise((resolve, reject) => { + const ws = fs.createWriteStream(`outputData.csv`, { flags: "w" }); + write(allCsvRows, { headers: true }) .pipe(ws) .on("finish", () => { - console.log(`CSV file has been written.`); - resolve; + console.log(`CSV file has been written.`); + resolve; }); + }); + }) + .catch((err) => { + console.error("Error reading CSV file:", err); }); -}).catch((err) => { - console.error('Error reading CSV file:', err); -}); diff --git a/adapters/nile/src/sdk/config.ts b/adapters/nile/src/sdk/config.ts index 9edc9576..acd04af6 100644 --- a/adapters/nile/src/sdk/config.ts +++ b/adapters/nile/src/sdk/config.ts @@ -6,8 +6,11 @@ export const SUBGRAPH_URL = export const client = createPublicClient({ chain: linea, - transport: http("https://rpc.linea.build", { - retryCount: 5, - timeout: 60_000, - }), + transport: http( + `https://linea-mainnet.infura.io/v3/${process.env.OPENBLOCK_LINEA_INFURA_API_KEY}`, + { + retryCount: 5, + timeout: 60_000, + }, + ), }); diff --git a/adapters/nile/src/sdk/lensDetails.ts b/adapters/nile/src/sdk/lensDetails.ts index b5d9934e..659452e0 100644 --- a/adapters/nile/src/sdk/lensDetails.ts +++ b/adapters/nile/src/sdk/lensDetails.ts @@ -15,80 +15,101 @@ export interface VoteResponse { export const fetchUserVotes = async ( blockNumber: bigint, - userAddress: string, + userAddresses: string[], ): Promise => { const publicClient = client; - const userBalanceCall = await multicall( + const balanceCalls = userAddresses.map((userAddress) => ({ + address: VE_NILE_ADDRESS, + name: "balanceOf", + params: [userAddress], + })); + + const userBalances = await batchMulticall( publicClient, veNILEAbi as Abi, - [ - { - address: VE_NILE_ADDRESS, - name: "balanceOf", - params: [userAddress], - }, - ], + balanceCalls, blockNumber, + 1000, + 2000, ); - const userBalance = userBalanceCall[0].result as number; - - if (userBalance === 0) return []; + const tokenCalls: any = []; + userBalances.forEach((balance, index) => { + const userAddress = userAddresses[index]; + const userBalance = balance.result as number; - const calls = []; - for (let i = 0; i < userBalance; i++) { - calls.push({ - address: VE_NILE_ADDRESS, - name: "tokenOfOwnerByIndex", - params: [userAddress, i], - }); - } + if (userBalance > 0) { + for (let i = 0; i < userBalance; i++) { + tokenCalls.push({ + address: VE_NILE_ADDRESS, + name: "tokenOfOwnerByIndex", + params: [userAddress, i], + }); + } + } + }); - const userTokensCalls = await multicall( + const userTokensCalls = await batchMulticall( publicClient, veNILEAbi as Abi, - calls, + tokenCalls, blockNumber, + 1000, + 200, ); - const detailsCall = userTokensCalls.map((call) => { - return { - address: VE_NILE_ADDRESS, - name: "locked", - params: [call.result], - }; - }); + const detailsCalls = userTokensCalls.map((call) => ({ + address: VE_NILE_ADDRESS, + name: "locked", + params: [call.result], + })); - const res = (await multicall( + const res = (await batchMulticall( publicClient, veNILEAbi as Abi, - detailsCall, + detailsCalls, blockNumber, + 1000, + 2000, )) as any; - return res.map((r: any) => { + return res.map((r: any, index: any) => { + const userAddress = userAddresses[Math.floor(index / tokenCalls.length)]; return { result: { amount: r.result[0], userAddress } }; }) as VoteResponse[]; }; -function multicall( +async function batchMulticall( publicClient: PublicClient, abi: Abi, calls: any[], blockNumber: bigint, + batchSize: number, + delay: number, ) { - const call: MulticallParameters = { - contracts: calls.map((call) => { - return { + const results = []; + + for (let i = 0; i < calls.length; i += batchSize) { + const batch = calls.slice(i, i + batchSize); + + const call: MulticallParameters = { + contracts: batch.map((call) => ({ address: call.address as Address, abi, functionName: call.name, args: call.params, - }; - }), - blockNumber, - }; + })), + blockNumber, + }; + + const res = await publicClient.multicall(call); + results.push(...res); + + if (i + batchSize < calls.length) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } - return publicClient.multicall(call); + return results; } diff --git a/adapters/nile/src/sdk/lockers.ts b/adapters/nile/src/sdk/lockers.ts new file mode 100644 index 00000000..606bbf61 --- /dev/null +++ b/adapters/nile/src/sdk/lockers.ts @@ -0,0 +1,26 @@ +import { SUBGRAPH_URL } from "./config"; + +export const getUserLockers = async ( + blockNumber: number, +): Promise => { + const query = `query { + userLocker( + id: "lockers" + block: { number: ${blockNumber} } + ) { + users + } + }`; + + const response = await fetch(SUBGRAPH_URL, { + method: "POST", + body: JSON.stringify({ query }), + headers: { "Content-Type": "application/json" }, + }); + + const { + data: { userLocker }, + } = await response.json(); + + return userLocker.users; +};