Skip to content

Commit

Permalink
batch nile user positions fetching
Browse files Browse the repository at this point in the history
  • Loading branch information
echovl committed Nov 1, 2024
1 parent a77d147 commit 5098d34
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 93 deletions.
2 changes: 1 addition & 1 deletion adapters/nile/hourly_blocks.csv
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
number,timestamp
3976979,1713974398
11449898,1730365248
81 changes: 34 additions & 47 deletions adapters/nile/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import fs from "fs";
import { write } from "fast-csv";
import csv from 'csv-parser';
import csv from "csv-parser";
import { BlockData, OutputSchemaRow, UserVote } from "./sdk/types";
import {
getV2UserPositionsAtBlock,
Expand All @@ -9,6 +9,7 @@ import {
import { getTimestampAtBlock } from "./sdk/common";
import { fetchUserVotes } from "./sdk/lensDetails";
import BigNumber from "bignumber.js";
import { getUserLockers } from "./sdk/lockers";

const NILE_ADDRESS = "0xAAAac83751090C6ea42379626435f805DDF54DC8".toLowerCase();

Expand Down Expand Up @@ -82,14 +83,8 @@ export const getUserLiquidityTVLByBlock = async ({
getV3UserPositionsAtBlock(blockNumber),
]);

const userAddresses = [...v2Positions, ...v3Positions]
.map((pos) => pos.user)
.reduce(
(prev, curr) => (prev.includes(curr) ? prev : [...prev, curr]),
[] as string[],
);

const userVotes = await getUserVotesTVLByBlock(blockNumber, userAddresses);
const userLockers = await getUserLockers(blockNumber);
const userVotes = await getUserVotesTVLByBlock(blockNumber, userLockers);

// combine v2 & v3
const combinedPositions = [...v2Positions, ...v3Positions];
Expand Down Expand Up @@ -142,24 +137,16 @@ export const getUserVotesTVLByBlock = async (
[userAddress: string]: BigNumber;
};

const batchSize = 300;
let userVotesResult: any[] = [];
for (let i = 0; i < userAddresses.length; i += batchSize) {
const batch = userAddresses.slice(i, i + batchSize);
userVotesResult = userVotesResult.concat(
await Promise.all(
batch.map((user) => fetchUserVotes(BigInt(blockNumber), user)),
),
);
}
const userVotesResult = await fetchUserVotes(
BigInt(blockNumber),
userAddresses,
);

for (const userFecthedVotes of userVotesResult) {
for (const userVote of userFecthedVotes) {
const userAddress = userVote.result.userAddress.toLowerCase();
tokenBalanceMap[userAddress] = BigNumber(
tokenBalanceMap[userAddress] ?? 0,
).plus(userVote.result.amount.toString());
}
const userAddress = userFecthedVotes.result.userAddress.toLowerCase();
tokenBalanceMap[userAddress] = BigNumber(
tokenBalanceMap[userAddress] ?? 0,
).plus(userFecthedVotes.result.amount.toString());
}

Object.entries(tokenBalanceMap).forEach(([userAddress, balance]) => {
Expand All @@ -178,54 +165,54 @@ export const getUserVotesTVLByBlock = async (
// console.log("Done");
// });


const readBlocksFromCSV = async (filePath: string): Promise<BlockData[]> => {
const blocks: BlockData[] = [];

await new Promise<void>((resolve, reject) => {
fs.createReadStream(filePath)
.pipe(csv()) // Specify the separator as '\t' for TSV files
.on('data', (row) => {
.on("data", (row) => {
const blockNumber = parseInt(row.number, 10);
const blockTimestamp = parseInt(row.timestamp, 10);
if (!isNaN(blockNumber) && blockTimestamp) {
blocks.push({ blockNumber: blockNumber, blockTimestamp });
}
})
.on('end', () => {
.on("end", () => {
resolve();
})
.on('error', (err) => {
.on("error", (err) => {
reject(err);
});
});

return blocks;
};

readBlocksFromCSV("hourly_blocks.csv")
.then(async (blocks: any[]) => {
console.log(blocks);
const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks

readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => {
console.log(blocks);
const allCsvRows: any[] = []; // Array to accumulate CSV rows for all blocks

for (const block of blocks) {
for (const block of blocks) {
try {
const result = await getUserTVLByBlock(block);
// Accumulate CSV rows for all blocks
allCsvRows.push(...result);
const result = await getUserTVLByBlock(block);
// Accumulate CSV rows for all blocks
allCsvRows.push(...result);
} catch (error) {
console.error(`An error occurred for block ${block}:`, error);
console.error(`An error occurred for block ${block}:`, error);
}
}
await new Promise((resolve, reject) => {
const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' });
write(allCsvRows, { headers: true })
}
await new Promise((resolve, reject) => {
const ws = fs.createWriteStream(`outputData.csv`, { flags: "w" });
write(allCsvRows, { headers: true })
.pipe(ws)
.on("finish", () => {
console.log(`CSV file has been written.`);
resolve;
console.log(`CSV file has been written.`);
resolve;
});
});
})
.catch((err) => {
console.error("Error reading CSV file:", err);
});
}).catch((err) => {
console.error('Error reading CSV file:', err);
});
11 changes: 7 additions & 4 deletions adapters/nile/src/sdk/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ export const SUBGRAPH_URL =

export const client = createPublicClient({
chain: linea,
transport: http("https://rpc.linea.build", {
retryCount: 5,
timeout: 60_000,
}),
transport: http(
`https://linea-mainnet.infura.io/v3/${process.env.OPENBLOCK_LINEA_INFURA_API_KEY}`,
{
retryCount: 5,
timeout: 60_000,
},
),
});
103 changes: 62 additions & 41 deletions adapters/nile/src/sdk/lensDetails.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,80 +15,101 @@ export interface VoteResponse {

export const fetchUserVotes = async (
blockNumber: bigint,
userAddress: string,
userAddresses: string[],
): Promise<VoteResponse[]> => {
const publicClient = client;

const userBalanceCall = await multicall(
const balanceCalls = userAddresses.map((userAddress) => ({
address: VE_NILE_ADDRESS,
name: "balanceOf",
params: [userAddress],
}));

const userBalances = await batchMulticall(
publicClient,
veNILEAbi as Abi,
[
{
address: VE_NILE_ADDRESS,
name: "balanceOf",
params: [userAddress],
},
],
balanceCalls,
blockNumber,
1000,
2000,
);

const userBalance = userBalanceCall[0].result as number;

if (userBalance === 0) return [];
const tokenCalls: any = [];
userBalances.forEach((balance, index) => {
const userAddress = userAddresses[index];
const userBalance = balance.result as number;

const calls = [];
for (let i = 0; i < userBalance; i++) {
calls.push({
address: VE_NILE_ADDRESS,
name: "tokenOfOwnerByIndex",
params: [userAddress, i],
});
}
if (userBalance > 0) {
for (let i = 0; i < userBalance; i++) {
tokenCalls.push({
address: VE_NILE_ADDRESS,
name: "tokenOfOwnerByIndex",
params: [userAddress, i],
});
}
}
});

const userTokensCalls = await multicall(
const userTokensCalls = await batchMulticall(
publicClient,
veNILEAbi as Abi,
calls,
tokenCalls,
blockNumber,
1000,
200,
);

const detailsCall = userTokensCalls.map((call) => {
return {
address: VE_NILE_ADDRESS,
name: "locked",
params: [call.result],
};
});
const detailsCalls = userTokensCalls.map((call) => ({
address: VE_NILE_ADDRESS,
name: "locked",
params: [call.result],
}));

const res = (await multicall(
const res = (await batchMulticall(
publicClient,
veNILEAbi as Abi,
detailsCall,
detailsCalls,
blockNumber,
1000,
2000,
)) as any;

return res.map((r: any) => {
return res.map((r: any, index: any) => {
const userAddress = userAddresses[Math.floor(index / tokenCalls.length)];
return { result: { amount: r.result[0], userAddress } };
}) as VoteResponse[];
};

function multicall(
async function batchMulticall(
publicClient: PublicClient,
abi: Abi,
calls: any[],
blockNumber: bigint,
batchSize: number,
delay: number,
) {
const call: MulticallParameters = {
contracts: calls.map((call) => {
return {
const results = [];

for (let i = 0; i < calls.length; i += batchSize) {
const batch = calls.slice(i, i + batchSize);

const call: MulticallParameters = {
contracts: batch.map((call) => ({
address: call.address as Address,
abi,
functionName: call.name,
args: call.params,
};
}),
blockNumber,
};
})),
blockNumber,
};

const res = await publicClient.multicall(call);
results.push(...res);

if (i + batchSize < calls.length) {
await new Promise((resolve) => setTimeout(resolve, delay));
}
}

return publicClient.multicall(call);
return results;
}
26 changes: 26 additions & 0 deletions adapters/nile/src/sdk/lockers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { SUBGRAPH_URL } from "./config";

export const getUserLockers = async (
blockNumber: number,
): Promise<string[]> => {
const query = `query {
userLocker(
id: "lockers"
block: { number: ${blockNumber} }
) {
users
}
}`;

const response = await fetch(SUBGRAPH_URL, {
method: "POST",
body: JSON.stringify({ query }),
headers: { "Content-Type": "application/json" },
});

const {
data: { userLocker },
} = await response.json();

return userLocker.users;
};

0 comments on commit 5098d34

Please sign in to comment.