From d5517d10d05ccfd464a53193fcd8782b08a33590 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 29 Nov 2024 03:42:43 +0000 Subject: [PATCH 01/17] init --- src/cli.ts | 35 +++++++-- src/order.ts | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/types.ts | 5 ++ 3 files changed, 240 insertions(+), 8 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 591c82e5..cd64fa5a 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,6 +26,8 @@ import { getOrderbookOwnersProfileMapFromSg, handleAddOrderbookOwnersProfileMap, handleRemoveOrderbookOwnersProfileMap, + downscaleProtection, + resetLimits, } from "./order"; import { diag, @@ -490,17 +492,24 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? ctx, ); + const orderbooksOwnersProfileMap = await getOrderbookOwnersProfileMapFromSg( + ordersDetails, + config.viemClient as any as ViemClient, + tokens, + (options as CliOptions).ownerProfile, + ); + await downscaleProtection( + orderbooksOwnersProfileMap, + config.viemClient as any as ViemClient, + options.ownerProfile, + ); + return { roundGap, options: options as CliOptions, poolUpdateInterval, config, - orderbooksOwnersProfileMap: await getOrderbookOwnersProfileMapFromSg( - ordersDetails, - config.viemClient as any as ViemClient, - tokens, - (options as CliOptions).ownerProfile, - ), + orderbooksOwnersProfileMap, tokens, lastReadOrdersTimestamp, }; @@ -802,6 +811,7 @@ export const main = async (argv: any, version?: string) => { startTime: lastReadOrdersTimestamp, }), ); + let ordersDidChange = false; const results = await Promise.allSettled( lastReadOrdersMap.map((v) => getOrderChanges( @@ -816,6 +826,9 @@ export const main = async (argv: any, version?: string) => { for (let i = 0; i < results.length; i++) { const res = results[i]; if (res.status === "fulfilled") { + if (res.value.addOrders.length || res.value.removeOrders.length) { + ordersDidChange = true; + } lastReadOrdersMap[i].skip += res.value.count; try { await handleAddOrderbookOwnersProfileMap( @@ -840,6 +853,16 @@ export const main = async (argv: any, version?: string) => { } } } + + // in case there are new orders or removed order, re evaluate owners limits + if (ordersDidChange) { + resetLimits(orderbooksOwnersProfileMap, options.ownerProfile); + await downscaleProtection( + orderbooksOwnersProfileMap, + config.viemClient as any as ViemClient, + options.ownerProfile, + ); + } } catch { /**/ } diff --git a/src/order.ts b/src/order.ts index d27bf8c0..6a7923ff 100644 --- a/src/order.ts +++ b/src/order.ts @@ -1,10 +1,10 @@ -import { OrderV3 } from "./abis"; +import { orderbookAbi, OrderV3 } from "./abis"; import { SgOrder } from "./query"; import { Span } from "@opentelemetry/api"; import { hexlify } from "ethers/lib/utils"; import { addWatchedToken } from "./account"; import { getTokenSymbol, shuffleArray } from "./utils"; -import { decodeAbiParameters, parseAbiParameters } from "viem"; +import { decodeAbiParameters, parseAbi, parseAbiParameters } from "viem"; import { Pair, Order, @@ -14,6 +14,10 @@ import { OrdersProfileMap, OwnersProfileMap, OrderbooksOwnersProfileMap, + TokensOwnersVaults, + OTOVMap, + OwnersVaults, + Vault, } from "./types"; /** @@ -383,3 +387,203 @@ function gatherPairs( } } } + +/** + * Builds a map with following form from an `OrderbooksOwnersProfileMap` instance: + * `orderbook -> token -> owner -> vaults` called `OTOVMap` + * This is later on used to evaluate the owners limits + */ +export function buildOtovMap(orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap): OTOVMap { + const result: OTOVMap = new Map(); + orderbooksOwnersProfileMap.forEach((ownersProfileMap, orderbook) => { + const tokensOwnersVaults: TokensOwnersVaults = new Map(); + ownersProfileMap.forEach((ownerProfile, owner) => { + ownerProfile.orders.forEach((orderProfile) => { + orderProfile.takeOrders.forEach((pair) => { + const token = pair.sellToken.toLowerCase(); + const vaultId = + pair.takeOrder.order.validOutputs[ + pair.takeOrder.outputIOIndex + ].vaultId.toLowerCase(); + const ownersVaults = tokensOwnersVaults.get(token); + if (ownersVaults) { + const vaults = ownersVaults.get(owner.toLowerCase()); + if (vaults) { + if (!vaults.find((v) => v.vaultId === vaultId)) + vaults.push({ vaultId, balance: 0n }); + } else { + ownersVaults.set(owner.toLowerCase(), [{ vaultId, balance: 0n }]); + } + } else { + const newOwnersVaults: OwnersVaults = new Map(); + newOwnersVaults.set(owner.toLowerCase(), [{ vaultId, balance: 0n }]); + tokensOwnersVaults.set(token, newOwnersVaults); + } + }); + }); + }); + result.set(orderbook, tokensOwnersVaults); + }); + return result; +} + +/** + * Gets vault balances of instance of an OrderbooksTokensOwnersVaults + */ +export async function fecthVaultBalances( + orderbooksTokensOwnersVaults: OTOVMap, + viemClient: ViemClient, + multicallAddressOverride?: string, +) { + const flattened: { + orderbook: string; + token: string; + owner: string; + vault: Vault; + }[] = []; + orderbooksTokensOwnersVaults.forEach((tokensOwnersVaults, orderbook) => { + tokensOwnersVaults.forEach((ownersVaults, token) => { + ownersVaults.forEach((vaults, owner) => { + vaults.forEach((v) => { + flattened.push({ + orderbook, + token, + owner, + vault: v, + }); + }); + }); + }); + }); + const multicallResult = await viemClient.multicall({ + multicallAddress: + (multicallAddressOverride as `0x${string}` | undefined) ?? + viemClient.chain?.contracts?.multicall3?.address, + allowFailure: false, + contracts: flattened.map((v) => ({ + address: v.orderbook as `0x${string}`, + allowFailure: false, + chainId: viemClient.chain!.id, + abi: parseAbi([orderbookAbi[3]]), + functionName: "vaultBalance", + args: [v.owner, v.token, v.vault.vaultId], + })), + }); + + for (let i = 0; i < multicallResult.length; i++) { + flattened[i].vault.balance = multicallResult[i]; + } +} + +/** + * Evaluates the owners limits by checking an owner vaults avg balances of a token against + * other owners total balances of that token to calculate a percentage, repeats the same + * process for every other token and owner and at the end ends up with map of owners with array + * of percentages, then calculates an avg of all those percenatges and that is applied as a divider + * factor to the owner's limit. + * This ensures that if an owner has many orders/vaults and has spread their balances across those + * many vaults and orders, he/she will get limited. + * Owners limits that are set by bot's admin as env or cli arg, are exluded from this evaluation process + */ +export function evaluateOwnersLimits( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + otovMap: OTOVMap, + ownerLimits?: Record, +) { + otovMap.forEach((tokensOwnersVaults, orderbook) => { + const ownersProfileMap = orderbooksOwnersProfileMap.get(orderbook); + if (ownersProfileMap) { + const ownersCuts: Map = new Map(); + tokensOwnersVaults.forEach((ownersVaults) => { + ownersVaults.forEach((vaults, owner) => { + // skip if owner limit is set by bot admin + if (typeof ownerLimits?.[owner.toLowerCase()] === "number") return; + + const ownerProfile = ownersProfileMap.get(owner); + if (ownerProfile) { + const avgBalance = + vaults.map((a) => a.balance).reduce((a, b) => a + b, 0n) / + BigInt(vaults.length); + const otherOwnersBalances = Array.from(ownersVaults) + .filter(([owner_]) => owner_ !== owner) + .flatMap(([, v]) => v.map((e) => e.balance)) + .reduce((a, b) => a + b, 0n); + const balanceRatioPercent = + otherOwnersBalances === 0n + ? 100n + : (avgBalance * 100n) / otherOwnersBalances; + + // divide into 4 segments + let ownerEvalDivideFactor = 1; + if (balanceRatioPercent >= 75n) { + ownerEvalDivideFactor = 1; + } else if (balanceRatioPercent >= 50n && balanceRatioPercent < 75n) { + ownerEvalDivideFactor = 2; + } else if (balanceRatioPercent >= 25n && balanceRatioPercent < 50n) { + ownerEvalDivideFactor = 3; + } else if (balanceRatioPercent > 0n && balanceRatioPercent < 25n) { + ownerEvalDivideFactor = 4; + } + + // gather owner divide factor for all of the owner's orders' tokens + // to calculate an avg from them all later on + const cuts = ownersCuts.get(owner.toLowerCase()); + if (cuts) { + cuts.push(ownerEvalDivideFactor); + } else { + ownersCuts.set(owner.toLowerCase(), [ownerEvalDivideFactor]); + } + } + }); + }); + ownersProfileMap.forEach((ownerProfile, owner) => { + const cuts = ownersCuts.get(owner); + if (cuts?.length) { + const avgCut = cuts.reduce((a, b) => a + b, 0) / cuts.length; + // round to nearest int, if turned out 0, set it to 1 as minimum + ownerProfile.limit = Math.round(ownerProfile.limit / avgCut); + if (ownerProfile.limit === 0) ownerProfile.limit = 1; + } + }); + } + }); +} + +/** + * Provides a protection by evaluating and possibly reducing owner's limit, + * this takes place by checking an owners avg vault balance of a token against + * all other owners cumulative balances, the calculated ratio is used a reducing + * factor for the owner limit when averaged out for all of tokens the owner has + */ +export async function downscaleProtection( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + viemClient: ViemClient, + ownerLimits?: Record, + multicallAddressOverride?: string, +) { + const otovMap = buildOtovMap(orderbooksOwnersProfileMap); + try { + await fecthVaultBalances(otovMap, viemClient, multicallAddressOverride); + evaluateOwnersLimits(orderbooksOwnersProfileMap, otovMap, ownerLimits); + } catch (error) { + /**/ + } +} + +/** + * Resets owners limit to default value + */ +export async function resetLimits( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + ownerLimits?: Record, +) { + orderbooksOwnersProfileMap.forEach((ownersProfileMap) => { + if (ownersProfileMap) { + ownersProfileMap.forEach((ownerProfile, owner) => { + // skip if owner limit is set by bot admin + if (typeof ownerLimits?.[owner.toLowerCase()] === "number") return; + ownerProfile.limit = DEFAULT_OWNER_LIMIT; + }); + } + }); +} diff --git a/src/types.ts b/src/types.ts index 0db97980..bd64be11 100644 --- a/src/types.ts +++ b/src/types.ts @@ -128,6 +128,11 @@ export type OrdersProfileMap = Map; export type OwnersProfileMap = Map; export type OrderbooksOwnersProfileMap = Map; +export type Vault = { vaultId: string; balance: bigint }; +export type OwnersVaults = Map; +export type TokensOwnersVaults = Map; +export type OTOVMap = Map; + export type ViemClient = WalletClient & PublicActions & { BALANCE: BigNumber; BOUNTY: TokenDetails[] }; From a10c4a8607421a6c16dbf7d1d37752a80aa32e88 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 29 Nov 2024 21:55:49 +0000 Subject: [PATCH 02/17] add tests --- src/order.ts | 4 +- test/cli.test.js | 4 +- test/orders.test.js | 214 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 217 insertions(+), 5 deletions(-) diff --git a/src/order.ts b/src/order.ts index 6a7923ff..e4cd921f 100644 --- a/src/order.ts +++ b/src/order.ts @@ -431,7 +431,7 @@ export function buildOtovMap(orderbooksOwnersProfileMap: OrderbooksOwnersProfile * Gets vault balances of instance of an OrderbooksTokensOwnersVaults */ export async function fecthVaultBalances( - orderbooksTokensOwnersVaults: OTOVMap, + otovMap: OTOVMap, viemClient: ViemClient, multicallAddressOverride?: string, ) { @@ -441,7 +441,7 @@ export async function fecthVaultBalances( owner: string; vault: Vault; }[] = []; - orderbooksTokensOwnersVaults.forEach((tokensOwnersVaults, orderbook) => { + otovMap.forEach((tokensOwnersVaults, orderbook) => { tokensOwnersVaults.forEach((ownersVaults, token) => { ownersVaults.forEach((vaults, owner) => { vaults.forEach((v) => { diff --git a/test/cli.test.js b/test/cli.test.js index 228c15ac..caad7711 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -201,8 +201,8 @@ describe("Test cli", async function () { route: "single", rpcRecords: { "https://rpc.ankr.com/polygon/": { - req: 1, - success: 1, + req: 2, + success: 2, failure: 0, cache: {}, }, diff --git a/test/orders.test.js b/test/orders.test.js index a2ea81ff..955e8646 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -14,9 +14,13 @@ const { getOrderPairs, prepareOrdersForRound, getOrderbookOwnersProfileMapFromSg, + buildOtovMap, + fecthVaultBalances, + evaluateOwnersLimits, + resetLimits, } = require("../src/order"); -describe("Test order details", async function () { +describe("Test order", async function () { beforeEach(() => mockServer.start(8081)); afterEach(() => mockServer.stop()); @@ -944,6 +948,214 @@ describe("Test order details", async function () { ]; assert.deepEqual(result4, expected4); }); + + it("should build OTOV map", async function () { + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [order1, order2] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), + getNewOrder(orderbook, owner2, token2, token1, 1), + ]; + + // build orderbook owner profile map + const ownerProfileMap = await getOrderbookOwnersProfileMapFromSg( + [order1, order2], + undefined, + [], + ); + + const result = buildOtovMap(ownerProfileMap); + const expected = new Map([ + [ + orderbook, + new Map([ + [ + token2.address, + new Map([[owner1, [{ vaultId: order1.outputs[0].vaultId, balance: 0n }]]]), + ], + [ + token1.address, + new Map([[owner2, [{ vaultId: order2.outputs[0].vaultId, balance: 0n }]]]), + ], + ]), + ], + ]); + + assert.deepEqual(result, expected); + }); + + it("should get vault balances for OTOV map", async function () { + // mock viem client + const viemClient = { + chain: { id: 137 }, + multicall: async () => [5n, 8n], + }; + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [order1, order2] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), + getNewOrder(orderbook, owner2, token2, token1, 1), + ]; + const otovMap = new Map([ + [ + orderbook, + new Map([ + [ + token2.address, + new Map([[owner1, [{ vaultId: order1.outputs[0].vaultId, balance: 0n }]]]), + ], + [ + token1.address, + new Map([[owner2, [{ vaultId: order2.outputs[0].vaultId, balance: 0n }]]]), + ], + ]), + ], + ]); + + await fecthVaultBalances(otovMap, viemClient); + const expected = new Map([ + [ + orderbook, + new Map([ + [ + token2.address, + new Map([[owner1, [{ vaultId: order1.outputs[0].vaultId, balance: 5n }]]]), + ], + [ + token1.address, + new Map([[owner2, [{ vaultId: order2.outputs[0].vaultId, balance: 8n }]]]), + ], + ]), + ], + ]); + + assert.deepEqual(otovMap, expected); + }); + + it("should evaluate owner limits", async function () { + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [owner1order1, owner2order1, owner1order2, owner2order2] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), + getNewOrder(orderbook, owner2, token1, token2, 1), + getNewOrder(orderbook, owner1, token2, token1, 1), + getNewOrder(orderbook, owner2, token2, token1, 1), + ]; + + // build orderbook owner profile map + const ownerProfileMap = await getOrderbookOwnersProfileMapFromSg( + [owner1order1, owner2order1, owner1order2, owner2order2], + undefined, + [], + { [owner1]: 4 }, // set owner1 limit as 4, owner2 unset (defaults to 25) + ); + const otovMap = new Map([ + [ + orderbook, + new Map([ + [ + token2.address, + new Map([ + [owner1, [{ vaultId: owner1order1.outputs[0].vaultId, balance: 5n }]], + [owner2, [{ vaultId: owner2order1.outputs[0].vaultId, balance: 5n }]], + ]), + ], + [ + token1.address, + new Map([ + [owner1, [{ vaultId: owner1order2.outputs[0].vaultId, balance: 9n }]], + [owner2, [{ vaultId: owner2order2.outputs[0].vaultId, balance: 2n }]], + ]), + ], + ]), + ], + ]); + evaluateOwnersLimits(ownerProfileMap, otovMap, { [owner1]: 4 }); + + // after evaluation, owner 2 limit should be reduced to 10 from the default 25, + // that is because owner2 relative to owner1 has 2/9 of the total token1 supply + // and has 1/1 of token2 supply, 2/9 goes into the bracket of 0 - 25%, ie divide factor + // of 4 and 1/1 goes into barcket of 75 - >100%, ie divide factor of 1, avg of the factors + // equals to: (1 + 4) / 2 = 2.5 and then the default owner2 limit which was 25, + // divided by 2/5 equals to 10 + // owner1 limit stays unchanged because it was set originally by the admin + const expected = await getOrderbookOwnersProfileMapFromSg( + [owner1order1, owner2order1, owner1order2, owner2order2], + undefined, + [], + { [owner1]: 4, [owner2]: 10 }, + ); + + assert.deepEqual(ownerProfileMap, expected); + }); + + it("should reset owners limit", async function () { + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [order1, order2] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), + getNewOrder(orderbook, owner2, token2, token1, 1), + ]; + + // build orderbook owner profile map + const ownerProfileMap = await getOrderbookOwnersProfileMapFromSg( + [order1, order2], + undefined, + [], + { [owner1]: 4, [owner2]: 10 }, // explicitly set owner2 limit to 10 for reset test + ); + // reset owner limits, only resets non admin set owner limit, ie only owner2 limit back to 25 + resetLimits(ownerProfileMap, { [owner1]: 4 }); + + const expected = await getOrderbookOwnersProfileMapFromSg([order1, order2], undefined, [], { + [owner1]: 4, + }); + assert.deepEqual(ownerProfileMap, expected); + }); }); function getOrderStruct(order) { From ec37cad7096cfd14788946009c7c1ee4e474b594 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 30 Nov 2024 00:04:32 +0000 Subject: [PATCH 03/17] update --- src/cli.ts | 23 +++----- src/order.ts | 107 +++++++++++++++++++----------------- test/cli.test.js | 4 +- test/orders.test.js | 131 +++++++++++++++++++++++++++----------------- 4 files changed, 148 insertions(+), 117 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index cd64fa5a..8cafc055 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -22,12 +22,11 @@ import { getBatchEthBalance, } from "./account"; import { + downscaleProtection, prepareOrdersForRound, getOrderbookOwnersProfileMapFromSg, handleAddOrderbookOwnersProfileMap, handleRemoveOrderbookOwnersProfileMap, - downscaleProtection, - resetLimits, } from "./order"; import { diag, @@ -492,24 +491,17 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? ctx, ); - const orderbooksOwnersProfileMap = await getOrderbookOwnersProfileMapFromSg( - ordersDetails, - config.viemClient as any as ViemClient, - tokens, - (options as CliOptions).ownerProfile, - ); - await downscaleProtection( - orderbooksOwnersProfileMap, - config.viemClient as any as ViemClient, - options.ownerProfile, - ); - return { roundGap, options: options as CliOptions, poolUpdateInterval, config, - orderbooksOwnersProfileMap, + orderbooksOwnersProfileMap: await getOrderbookOwnersProfileMapFromSg( + ordersDetails, + config.viemClient as any as ViemClient, + tokens, + (options as CliOptions).ownerProfile, + ), tokens, lastReadOrdersTimestamp, }; @@ -856,7 +848,6 @@ export const main = async (argv: any, version?: string) => { // in case there are new orders or removed order, re evaluate owners limits if (ordersDidChange) { - resetLimits(orderbooksOwnersProfileMap, options.ownerProfile); await downscaleProtection( orderbooksOwnersProfileMap, config.viemClient as any as ViemClient, diff --git a/src/order.ts b/src/order.ts index e4cd921f..02f2bc32 100644 --- a/src/order.ts +++ b/src/order.ts @@ -4,7 +4,7 @@ import { Span } from "@opentelemetry/api"; import { hexlify } from "ethers/lib/utils"; import { addWatchedToken } from "./account"; import { getTokenSymbol, shuffleArray } from "./utils"; -import { decodeAbiParameters, parseAbi, parseAbiParameters } from "viem"; +import { decodeAbiParameters, erc20Abi, parseAbi, parseAbiParameters } from "viem"; import { Pair, Order, @@ -428,50 +428,33 @@ export function buildOtovMap(orderbooksOwnersProfileMap: OrderbooksOwnersProfile } /** - * Gets vault balances of instance of an OrderbooksTokensOwnersVaults + * Gets vault balances of an owner's vaults of a given token */ -export async function fecthVaultBalances( - otovMap: OTOVMap, +export async function fetchVaultBalances( + orderbook: string, + token: string, + owner: string, + vaults: Vault[], viemClient: ViemClient, multicallAddressOverride?: string, ) { - const flattened: { - orderbook: string; - token: string; - owner: string; - vault: Vault; - }[] = []; - otovMap.forEach((tokensOwnersVaults, orderbook) => { - tokensOwnersVaults.forEach((ownersVaults, token) => { - ownersVaults.forEach((vaults, owner) => { - vaults.forEach((v) => { - flattened.push({ - orderbook, - token, - owner, - vault: v, - }); - }); - }); - }); - }); const multicallResult = await viemClient.multicall({ multicallAddress: (multicallAddressOverride as `0x${string}` | undefined) ?? viemClient.chain?.contracts?.multicall3?.address, allowFailure: false, - contracts: flattened.map((v) => ({ - address: v.orderbook as `0x${string}`, + contracts: vaults.map((v) => ({ + address: orderbook as `0x${string}`, allowFailure: false, chainId: viemClient.chain!.id, abi: parseAbi([orderbookAbi[3]]), functionName: "vaultBalance", - args: [v.owner, v.token, v.vault.vaultId], + args: [owner, token, v.vaultId], })), }); for (let i = 0; i < multicallResult.length; i++) { - flattened[i].vault.balance = multicallResult[i]; + vaults[i].balance = multicallResult[i]; } } @@ -485,29 +468,48 @@ export async function fecthVaultBalances( * many vaults and orders, he/she will get limited. * Owners limits that are set by bot's admin as env or cli arg, are exluded from this evaluation process */ -export function evaluateOwnersLimits( +export async function evaluateOwnersLimits( orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, otovMap: OTOVMap, + viemClient: ViemClient, ownerLimits?: Record, + multicallAddressOverride?: string, ) { - otovMap.forEach((tokensOwnersVaults, orderbook) => { + for (const [orderbook, tokensOwnersVaults] of otovMap) { const ownersProfileMap = orderbooksOwnersProfileMap.get(orderbook); if (ownersProfileMap) { const ownersCuts: Map = new Map(); - tokensOwnersVaults.forEach((ownersVaults) => { - ownersVaults.forEach((vaults, owner) => { + for (const [token, ownersVaults] of tokensOwnersVaults) { + const obTokenBalance = await viemClient.readContract({ + address: token as `0x${string}`, + abi: erc20Abi, + functionName: "balanceOf", + args: [orderbook as `0x${string}`], + }); + for (const [owner, vaults] of ownersVaults) { // skip if owner limit is set by bot admin - if (typeof ownerLimits?.[owner.toLowerCase()] === "number") return; + if (typeof ownerLimits?.[owner.toLowerCase()] === "number") continue; const ownerProfile = ownersProfileMap.get(owner); if (ownerProfile) { - const avgBalance = - vaults.map((a) => a.balance).reduce((a, b) => a + b, 0n) / - BigInt(vaults.length); - const otherOwnersBalances = Array.from(ownersVaults) - .filter(([owner_]) => owner_ !== owner) - .flatMap(([, v]) => v.map((e) => e.balance)) - .reduce((a, b) => a + b, 0n); + await fetchVaultBalances( + orderbook, + token, + owner, + vaults, + viemClient, + multicallAddressOverride, + ); + const ownerTotalBalance = vaults.reduce( + (a, b) => ({ + balance: a.balance + b.balance, + }), + { + balance: 0n, + }, + ).balance; + const avgBalance = ownerTotalBalance / BigInt(vaults.length); + const otherOwnersBalances = obTokenBalance - ownerTotalBalance; const balanceRatioPercent = otherOwnersBalances === 0n ? 100n @@ -534,8 +536,9 @@ export function evaluateOwnersLimits( ownersCuts.set(owner.toLowerCase(), [ownerEvalDivideFactor]); } } - }); - }); + } + } + ownersProfileMap.forEach((ownerProfile, owner) => { const cuts = ownersCuts.get(owner); if (cuts?.length) { @@ -546,10 +549,11 @@ export function evaluateOwnersLimits( } }); } - }); + } } /** + * This is a wrapper fn around evaluating owers limits. * Provides a protection by evaluating and possibly reducing owner's limit, * this takes place by checking an owners avg vault balance of a token against * all other owners cumulative balances, the calculated ratio is used a reducing @@ -559,15 +563,20 @@ export async function downscaleProtection( orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, viemClient: ViemClient, ownerLimits?: Record, + reset = true, multicallAddressOverride?: string, ) { - const otovMap = buildOtovMap(orderbooksOwnersProfileMap); - try { - await fecthVaultBalances(otovMap, viemClient, multicallAddressOverride); - evaluateOwnersLimits(orderbooksOwnersProfileMap, otovMap, ownerLimits); - } catch (error) { - /**/ + if (reset) { + resetLimits(orderbooksOwnersProfileMap, ownerLimits); } + const otovMap = buildOtovMap(orderbooksOwnersProfileMap); + await evaluateOwnersLimits( + orderbooksOwnersProfileMap, + otovMap, + viemClient, + ownerLimits, + multicallAddressOverride, + ); } /** diff --git a/test/cli.test.js b/test/cli.test.js index caad7711..228c15ac 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -201,8 +201,8 @@ describe("Test cli", async function () { route: "single", rpcRecords: { "https://rpc.ankr.com/polygon/": { - req: 2, - success: 2, + req: 1, + success: 1, failure: 0, cache: {}, }, diff --git a/test/orders.test.js b/test/orders.test.js index 955e8646..e6546385 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -15,9 +15,10 @@ const { prepareOrdersForRound, getOrderbookOwnersProfileMapFromSg, buildOtovMap, - fecthVaultBalances, + fetchVaultBalances, evaluateOwnersLimits, resetLimits, + downscaleProtection, } = require("../src/order"); describe("Test order", async function () { @@ -995,66 +996,46 @@ describe("Test order", async function () { assert.deepEqual(result, expected); }); - it("should get vault balances for OTOV map", async function () { + it("should get vault balances for owners vaults", async function () { // mock viem client const viemClient = { chain: { id: 137 }, - multicall: async () => [5n, 8n], + multicall: async () => [8n, 3n, 5n], }; const orderbook = hexlify(randomBytes(20)).toLowerCase(); - const owner1 = hexlify(randomBytes(20)).toLowerCase(); - const owner2 = hexlify(randomBytes(20)).toLowerCase(); - const token1 = { + const owner = hexlify(randomBytes(20)).toLowerCase(); + const token = { address: hexlify(randomBytes(20)).toLowerCase(), decimals: 6, symbol: "NewToken1", }; - const token2 = { - address: hexlify(randomBytes(20)).toLowerCase(), - decimals: 6, - symbol: "NewToken1", - }; - const [order1, order2] = [ - getNewOrder(orderbook, owner1, token1, token2, 1), - getNewOrder(orderbook, owner2, token2, token1, 1), + const vaults = [ + { vaultId: "1", balance: 0n }, + { vaultId: "2", balance: 0n }, + { vaultId: "3", balance: 0n }, + ]; + await fetchVaultBalances(orderbook, token.address, owner, vaults, viemClient); + const expected = [ + { vaultId: "1", balance: 8n }, + { vaultId: "2", balance: 3n }, + { vaultId: "3", balance: 5n }, ]; - const otovMap = new Map([ - [ - orderbook, - new Map([ - [ - token2.address, - new Map([[owner1, [{ vaultId: order1.outputs[0].vaultId, balance: 0n }]]]), - ], - [ - token1.address, - new Map([[owner2, [{ vaultId: order2.outputs[0].vaultId, balance: 0n }]]]), - ], - ]), - ], - ]); - - await fecthVaultBalances(otovMap, viemClient); - const expected = new Map([ - [ - orderbook, - new Map([ - [ - token2.address, - new Map([[owner1, [{ vaultId: order1.outputs[0].vaultId, balance: 5n }]]]), - ], - [ - token1.address, - new Map([[owner2, [{ vaultId: order2.outputs[0].vaultId, balance: 8n }]]]), - ], - ]), - ], - ]); - assert.deepEqual(otovMap, expected); + assert.deepEqual(vaults, expected); }); it("should evaluate owner limits", async function () { + // mock viem client + let counter = -1; + const viemClient = { + chain: { id: 137 }, + readContract: async () => 10n, + multicall: async () => { + counter++; + if (counter === 0) return [5n]; // for tkn1 owner2 + if (counter === 1) return [1n]; // for tkn2 owner2 + }, + }; const orderbook = hexlify(randomBytes(20)).toLowerCase(); const owner1 = hexlify(randomBytes(20)).toLowerCase(); const owner2 = hexlify(randomBytes(20)).toLowerCase(); @@ -1097,17 +1078,17 @@ describe("Test order", async function () { token1.address, new Map([ [owner1, [{ vaultId: owner1order2.outputs[0].vaultId, balance: 9n }]], - [owner2, [{ vaultId: owner2order2.outputs[0].vaultId, balance: 2n }]], + [owner2, [{ vaultId: owner2order2.outputs[0].vaultId, balance: 1n }]], ]), ], ]), ], ]); - evaluateOwnersLimits(ownerProfileMap, otovMap, { [owner1]: 4 }); + await evaluateOwnersLimits(ownerProfileMap, otovMap, viemClient, { [owner1]: 4 }); // after evaluation, owner 2 limit should be reduced to 10 from the default 25, // that is because owner2 relative to owner1 has 2/9 of the total token1 supply - // and has 1/1 of token2 supply, 2/9 goes into the bracket of 0 - 25%, ie divide factor + // and has 1/1 of token2 supply, 1/9 goes into the bracket of 0 - 25%, ie divide factor // of 4 and 1/1 goes into barcket of 75 - >100%, ie divide factor of 1, avg of the factors // equals to: (1 + 4) / 2 = 2.5 and then the default owner2 limit which was 25, // divided by 2/5 equals to 10 @@ -1156,6 +1137,56 @@ describe("Test order", async function () { }); assert.deepEqual(ownerProfileMap, expected); }); + + it("should run downscaleProtection", async function () { + // mock viem client + let counter = -1; + const viemClient = { + chain: { id: 137 }, + readContract: async () => 10n, + multicall: async () => { + counter++; + if (counter === 0) return [5n]; // for tkn1 owner2 + if (counter === 1) return [1n]; // for tkn2 owner2 + }, + }; + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [owner1order1, owner2order1, owner1order2, owner2order2] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), + getNewOrder(orderbook, owner2, token1, token2, 1), + getNewOrder(orderbook, owner1, token2, token1, 1), + getNewOrder(orderbook, owner2, token2, token1, 1), + ]; + + // build orderbook owner profile map + const ownerProfileMap = await getOrderbookOwnersProfileMapFromSg( + [owner1order1, owner2order1, owner1order2, owner2order2], + undefined, + [], + { [owner1]: 4 }, + ); + await downscaleProtection(ownerProfileMap, viemClient, { [owner1]: 4 }); + const expected = await getOrderbookOwnersProfileMapFromSg( + [owner1order1, owner2order1, owner1order2, owner2order2], + undefined, + [], + { [owner1]: 4, [owner2]: 10 }, + ); + + assert.deepEqual(ownerProfileMap, expected); + }); }); function getOrderStruct(order) { From dcf842f4a262117dd1f8dd260b5e5117340c4d7d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 30 Nov 2024 00:58:00 +0000 Subject: [PATCH 04/17] update --- src/order.ts | 64 ++++++++++++++++++++++++++------------------- src/types.ts | 2 +- test/orders.test.js | 13 +++++---- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/order.ts b/src/order.ts index 02f2bc32..7ec7fda0 100644 --- a/src/order.ts +++ b/src/order.ts @@ -1,23 +1,23 @@ -import { orderbookAbi, OrderV3 } from "./abis"; import { SgOrder } from "./query"; import { Span } from "@opentelemetry/api"; import { hexlify } from "ethers/lib/utils"; import { addWatchedToken } from "./account"; +import { orderbookAbi, OrderV3 } from "./abis"; import { getTokenSymbol, shuffleArray } from "./utils"; import { decodeAbiParameters, erc20Abi, parseAbi, parseAbiParameters } from "viem"; import { Pair, Order, + Vault, + OTOVMap, ViemClient, + OwnersVaults, TokenDetails, BundledOrders, OrdersProfileMap, OwnersProfileMap, - OrderbooksOwnersProfileMap, TokensOwnersVaults, - OTOVMap, - OwnersVaults, - Vault, + OrderbooksOwnersProfileMap, } from "./types"; /** @@ -51,6 +51,7 @@ export function toOrder(orderLog: any): Order { * Get all pairs of an order */ export async function getOrderPairs( + orderHash: string, orderStruct: Order, viemClient: ViemClient, tokens: TokenDetails[], @@ -116,10 +117,13 @@ export async function getOrderPairs( sellTokenSymbol: _outputSymbol, sellTokenDecimals: _output.decimals, takeOrder: { - order: orderStruct, - inputIOIndex: k, - outputIOIndex: j, - signedContext: [], + id: orderHash, + takeOrder: { + order: orderStruct, + inputIOIndex: k, + outputIOIndex: j, + signedContext: [], + }, }, }); } @@ -141,6 +145,7 @@ export async function handleAddOrderbookOwnersProfileMap( const changes: Record = {}; for (let i = 0; i < ordersDetails.length; i++) { const orderDetails = ordersDetails[i]; + const orderHash = orderDetails.orderHash.toLowerCase(); const orderbook = orderDetails.orderbook.id.toLowerCase(); const orderStruct = toOrder( decodeAbiParameters( @@ -158,12 +163,13 @@ export async function handleAddOrderbookOwnersProfileMap( if (orderbookOwnerProfileItem) { const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); if (ownerProfile) { - const order = ownerProfile.orders.get(orderDetails.orderHash.toLowerCase()); + const order = ownerProfile.orders.get(orderHash); if (!order) { - ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { + ownerProfile.orders.set(orderHash, { active: true, order: orderStruct, takeOrders: await getOrderPairs( + orderHash, orderStruct, viemClient, tokens, @@ -176,10 +182,16 @@ export async function handleAddOrderbookOwnersProfileMap( } } else { const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + ordersProfileMap.set(orderHash, { active: true, order: orderStruct, - takeOrders: await getOrderPairs(orderStruct, viemClient, tokens, orderDetails), + takeOrders: await getOrderPairs( + orderHash, + orderStruct, + viemClient, + tokens, + orderDetails, + ), consumedTakeOrders: [], }); orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { @@ -189,10 +201,16 @@ export async function handleAddOrderbookOwnersProfileMap( } } else { const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + ordersProfileMap.set(orderHash, { active: true, order: orderStruct, - takeOrders: await getOrderPairs(orderStruct, viemClient, tokens, orderDetails), + takeOrders: await getOrderPairs( + orderHash, + orderStruct, + viemClient, + tokens, + orderDetails, + ), consumedTakeOrders: [], }); const ownerProfileMap: OwnersProfileMap = new Map(); @@ -363,10 +381,7 @@ function gatherPairs( if ( !bundleOrder.takeOrders.find((v) => v.id.toLowerCase() === orderHash.toLowerCase()) ) { - bundleOrder.takeOrders.push({ - id: orderHash, - takeOrder: pair.takeOrder, - }); + bundleOrder.takeOrders.push(pair.takeOrder); } } else { bundledOrders.push({ @@ -377,12 +392,7 @@ function gatherPairs( sellToken: pair.sellToken, sellTokenDecimals: pair.sellTokenDecimals, sellTokenSymbol: pair.sellTokenSymbol, - takeOrders: [ - { - id: orderHash, - takeOrder: pair.takeOrder, - }, - ], + takeOrders: [pair.takeOrder], }); } } @@ -402,8 +412,8 @@ export function buildOtovMap(orderbooksOwnersProfileMap: OrderbooksOwnersProfile orderProfile.takeOrders.forEach((pair) => { const token = pair.sellToken.toLowerCase(); const vaultId = - pair.takeOrder.order.validOutputs[ - pair.takeOrder.outputIOIndex + pair.takeOrder.takeOrder.order.validOutputs[ + pair.takeOrder.takeOrder.outputIOIndex ].vaultId.toLowerCase(); const ownersVaults = tokensOwnersVaults.get(token); if (ownersVaults) { diff --git a/src/types.ts b/src/types.ts index bd64be11..7b0884cb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -112,7 +112,7 @@ export type Pair = { sellToken: string; sellTokenDecimals: number; sellTokenSymbol: string; - takeOrder: TakeOrder; + takeOrder: TakeOrderDetails; }; export type OrderProfile = { active: boolean; diff --git a/test/orders.test.js b/test/orders.test.js index e6546385..980438b9 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -619,7 +619,7 @@ describe("Test order", async function () { const orderStruct = toOrder( decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], ); - const result = await getOrderPairs(orderStruct, undefined, [], order1); + const result = await getOrderPairs(order1.orderHash, orderStruct, undefined, [], order1); const expected = [ { buyToken: orderStruct.validInputs[0].token, @@ -629,10 +629,13 @@ describe("Test order", async function () { sellTokenSymbol: order1.outputs[0].token.symbol, sellTokenDecimals: orderStruct.validOutputs[0].decimals, takeOrder: { - order: orderStruct, - inputIOIndex: 0, - outputIOIndex: 0, - signedContext: [], + id: order1.orderHash, + takeOrder: { + order: orderStruct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, }, }, ]; From 53000b1eea9651cc4402ca1593c447e5fe2e085c Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 2 Dec 2024 03:20:58 +0000 Subject: [PATCH 05/17] init --- .eslintrc.json | 1 + README.md | 4 +- example.env | 2 +- package-lock.json | 18 +++++++-- package.json | 7 ++-- src/cli.ts | 4 +- src/config.ts | 17 ++++++++ src/gas.ts | 80 +++++++++++++++++++++++++++++++++++++ src/modes/index.ts | 25 +++++++++++- src/modes/interOrderbook.ts | 35 +++++++++++----- src/modes/intraOrderbook.ts | 33 ++++++++++----- src/modes/routeProcessor.ts | 41 +++++++++++++++---- src/processOrders.ts | 22 +++++++--- src/types.ts | 2 + test/findOpp.test.js | 3 +- test/gas.test.ts | 61 ++++++++++++++++++++++++++++ test/processPair.test.js | 6 ++- 17 files changed, 313 insertions(+), 48 deletions(-) create mode 100644 src/gas.ts create mode 100644 test/gas.test.ts diff --git a/.eslintrc.json b/.eslintrc.json index 1e3c85fa..bb3e6849 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -26,6 +26,7 @@ "@typescript-eslint/no-non-null-assertion": "off", "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-namespace": "off", "semi": [ "error", "always" ], "prettier/prettier": [ "error", diff --git a/README.md b/README.md index 19376695..ce610a66 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Other optional arguments are: - `--owner-profile`, Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables - `--public-rpc`, Allows to use public RPCs as fallbacks, default is false. Will override the 'PUBLIC_RPC' in env variables - `--gas-price-multiplier`, Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7%. Will override the 'GAS_PRICE_MULTIPLIER' in env variables -- `--gas-limit-multiplier`, Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables +- `--gas-limit-multiplier`, Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables - `--tx-gas`, Option to set a static gas limit for all submitting txs. Will override the 'TX_GAS' in env variables - `-V` or `--version`, output the version number - `-h` or `--help`, output usage information @@ -262,7 +262,7 @@ ROUTE="single" # Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7% GAS_PRICE_MULTIPLIER= -# Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8% +# Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5% GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs diff --git a/example.env b/example.env index c3b9a355..61d8b14a 100644 --- a/example.env +++ b/example.env @@ -88,7 +88,7 @@ ROUTE="single" # Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7% GAS_PRICE_MULTIPLIER= -# Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8% +# Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5% GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs diff --git a/package-lock.json b/package-lock.json index 221431f6..e083100b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "@nomicfoundation/hardhat-viem": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.0", "@nomiclabs/hardhat-waffle": "2.0.3", + "@types/chai": "^4.2.0", "@types/mocha": "^10.0.0", "@types/node": "^18.13.0", "@typescript-eslint/eslint-plugin": "^5.42.0", @@ -4059,9 +4060,9 @@ } }, "node_modules/@types/chai": { - "version": "4.3.16", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.16.tgz", - "integrity": "sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.0.tgz", + "integrity": "sha512-zw8UvoBEImn392tLjxoavuonblX/4Yb9ha4KBU10FirCfwgzhKO0dvyJSF9ByxV1xK1r2AgnAi/tvQaLgxQqxA==", "dev": true }, "node_modules/@types/chai-as-promised": { @@ -14263,6 +14264,7 @@ "hasInstallScript": true, "inBundle": true, "license": "MIT", + "peer": true, "dependencies": { "node-addon-api": "^2.0.0", "node-gyp-build": "^4.2.0" @@ -14892,7 +14894,8 @@ "version": "2.0.2", "dev": true, "inBundle": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/ganache-core/node_modules/node-fetch": { "version": "2.1.2", @@ -14908,6 +14911,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -25150,6 +25154,12 @@ } } }, + "node_modules/vitest/node_modules/@types/chai": { + "version": "4.3.20", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.20.tgz", + "integrity": "sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ==", + "dev": true + }, "node_modules/walk-back": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/walk-back/-/walk-back-5.1.0.tgz", diff --git a/package.json b/package.json index 8f37bd6f..43e665ec 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "scripts": { "prepublish": "npm run build", "test": "npm run unit-test && npm run e2e-test", - "unit-test": "hardhat test ./test/*.test.js", + "unit-test": "hardhat test ./test/*.test.*", "e2e-test": "hardhat test", "lint": "eslint ./src ./test ./arb-bot.js", "lint-fix": "eslint ./src ./test ./arb-bot.js --fix", @@ -61,15 +61,16 @@ "@nomicfoundation/hardhat-viem": "^2.0.2", "@nomiclabs/hardhat-ethers": "^2.0.0", "@nomiclabs/hardhat-waffle": "2.0.3", + "@types/chai": "^4.2.0", "@types/mocha": "^10.0.0", "@types/node": "^18.13.0", "@typescript-eslint/eslint-plugin": "^5.42.0", "@typescript-eslint/parser": "^5.42.0", "chai": "^4.3.6", "copyfiles": "^2.4.1", + "eslint": "^8.26.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.1", - "eslint": "^8.26.0", "hardhat": "^2.17.8", "hardhat-contract-sizer": "^2.1.1", "hardhat-gas-reporter": "^1.0.4", @@ -78,8 +79,8 @@ "jsdoc-to-markdown": "^7.1.1", "mocha": "^10.1.0", "mockttp": "^3.12.0", - "rimraf": "^5.0.0", "prettier": "^3.1.1", + "rimraf": "^5.0.0", "ts-node": "^10.9.1", "typescript": "^5.0.4" } diff --git a/src/cli.ts b/src/cli.ts index 591c82e5..8e77a27c 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -194,7 +194,7 @@ const getOptions = async (argv: any, version?: string) => { ) .option( "--gas-limit-multiplier ", - "Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables", + "Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables", ) .option( "--tx-gas ", @@ -446,7 +446,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? throw "invalid gasLimitMultiplier value, must be an integer greater than zero"; } else throw "invalid gasLimitMultiplier value, must be an integer greater than zero"; } else { - options.gasLimitMultiplier = 108; + options.gasLimitMultiplier = 105; } if (options.txGas) { if (typeof options.txGas === "number") { diff --git a/src/config.ts b/src/config.ts index 035bc50f..c6cc304c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -58,6 +58,7 @@ export function getChainConfig(chainId: ChainId): ChainConfig { nativeWrappedToken, routeProcessors, stableTokens, + isL2: L2Chains.is(chain.id), }; } @@ -282,6 +283,22 @@ export function getWithdrawEnsureBytecode( return `0x0000000000000000000000000000000000000000000000000000000000000009${input}${bot}${inputBalance}${inputPrice}${output}${outputBalance}${outputPrice}${minimum}936d696e696d756d53656e6465724f757470757400000000000000000000000000000000000000000000000000000000000000000000000000000000000000530100001307000001100008011000070110000601100005011000010110000411120000471200003d1200000110000301100002011000010110000011120000471200003d1200002b120000211200001d020000`; } +/** + * List of L2 chains that require SEPARATE L1 gas actions. + * other L2 chains that dont require separate L1 gas actions + * such as Arbitrum and Polygon zkEvm are excluded, these chains' + * gas actions are performed the same as usual L1 chains. + */ +export enum L2Chains { + BASE = ChainId.BASE, + OPTIMISM = ChainId.OPTIMISM, +} +export namespace L2Chains { + export function is(chainId: number): boolean { + return Object.values(L2Chains).includes(chainId as any); + } +} + /** * Get meta info for a bot to post on otel */ diff --git a/src/gas.ts b/src/gas.ts new file mode 100644 index 00000000..c65b40ad --- /dev/null +++ b/src/gas.ts @@ -0,0 +1,80 @@ +import { BotConfig, RawTx, ViemClient } from "./types"; +import { publicActionsL2, walletActionsL2 } from "viem/op-stack"; + +/** + * Estimates gas cost of the given tx, also takes into account L1 gas cost if operating chain is L2. + * Not all L2 chains need to calculate L1 gas separately, some chains like Arbitrum and Polygon zkEvm, + * dont actually need anything extra other than usual gas estimation and they actually contain L1 gas in + * their usual gas estimation opertaion, but some other L2 chains such as Base and Optimism, do need to + * estimate L1 gas separately, so we will use a try/catch block + */ +export async function estimateGasCost( + tx: RawTx, + signer: ViemClient, + config: BotConfig, + l1GasPrice?: bigint, + l1Signer?: any, +) { + const gasPrice = tx.gasPrice ?? (await signer.getGasPrice()); + const gas = await signer.estimateGas(tx); + if (config.isL2) { + try { + const l1Signer_ = l1Signer + ? l1Signer + : signer.extend(walletActionsL2()).extend(publicActionsL2()); + if (typeof l1GasPrice !== "bigint") { + l1GasPrice = (await l1Signer_.getL1BaseFee()) as bigint; + } + const l1Gas = await l1Signer_.estimateL1Gas({ + to: tx.to, + data: tx.data, + }); + return { + gas, + gasPrice, + l1Gas, + l1GasPrice, + l1Cost: l1Gas * l1GasPrice, + totalGasCost: gasPrice * gas + l1Gas * l1GasPrice, + }; + } catch { + return { + gas, + gasPrice, + l1Gas: 0n, + l1GasPrice: 0n, + l1Cost: 0n, + totalGasCost: gasPrice * gas, + }; + } + } else { + return { + gas, + gasPrice, + l1Gas: 0n, + l1GasPrice: 0n, + l1Cost: 0n, + totalGasCost: gasPrice * gas, + }; + } +} + +/** + * Retruns the L1 gas cost of a transaction if operating chain is L2 else returns 0. + * Not all L2 chains need report the L1 gas separately to the usual transaction receipt, chains + * like Arbitrum and Polygon zkEvm report the tx used gas normally like any other L1 chain, but + * some other L2 chains like Base and Optimism report the L1 gas separately to L2 using the properties + * below, so we need to explicitly check for the, if they are not present in the receipt, then simply + * return 0 + */ +export function getL1Fee(receipt: any, config: BotConfig): bigint { + if (!config.isL2) return 0n; + + if (typeof receipt.l1Fee === "bigint") { + return receipt.l1Fee; + } else if (typeof receipt.l1GasPrice === "bigint" && typeof receipt.l1GasUsed === "bigint") { + return (receipt.l1GasPrice as bigint) * (receipt.l1GasUsed as bigint); + } else { + return 0n; + } +} diff --git a/src/modes/index.ts b/src/modes/index.ts index 92cb2b82..43dd8e63 100644 --- a/src/modes/index.ts +++ b/src/modes/index.ts @@ -1,9 +1,10 @@ -import { BigNumber, Contract } from "ethers"; import { PublicClient } from "viem"; import { DataFetcher } from "sushi"; import { Token } from "sushi/currency"; +import { BigNumber, Contract } from "ethers"; import { findOpp as findInterObOpp } from "./interOrderbook"; import { findOpp as findIntraObOpp } from "./intraOrderbook"; +import { publicActionsL2, walletActionsL2 } from "viem/op-stack"; import { findOppWithRetries as findRpOpp } from "./routeProcessor"; import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; @@ -51,6 +52,22 @@ export async function findOpp({ } catch { /**/ } + + // if chain is L2, get L1 gas price before dryruns + let l1Signer; + let l1GasPrice: bigint | undefined; + if (config.isL2) { + try { + // as already known, not all L2 chains support this method such as Arbitrum, + // only certain ones do, such as Base and Optimism, so use try/catch block + // and set to 0 on catch block so it becomes ineffective + l1Signer = signer.extend(walletActionsL2()).extend(publicActionsL2()); + l1GasPrice = await l1Signer.getL1BaseFee(); + } catch { + l1GasPrice = 0n; + } + } + const promises = [ findRpOpp({ orderPairObject, @@ -63,6 +80,8 @@ export async function findOpp({ ethPrice: inputToEthPrice, config, viemClient, + l1GasPrice, + l1Signer, }), findIntraObOpp({ orderPairObject, @@ -73,6 +92,8 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1GasPrice, + l1Signer, }), findInterObOpp({ orderPairObject, @@ -84,6 +105,8 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1GasPrice, + l1Signer, }), ]; const allResults = await Promise.allSettled(promises); diff --git a/src/modes/interOrderbook.ts b/src/modes/interOrderbook.ts index 1577f21e..d85bdd05 100644 --- a/src/modes/interOrderbook.ts +++ b/src/modes/interOrderbook.ts @@ -1,10 +1,11 @@ import { orderbookAbi } from "../abis"; +import { estimateGasCost } from "../gas"; import { BaseError, PublicClient } from "viem"; import { getBountyEnsureBytecode } from "../config"; import { BigNumber, Contract, ethers } from "ethers"; import { containsNodeError, errorSnapshot } from "../error"; import { estimateProfit, withBigintSerializer } from "../utils"; -import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; +import { BotConfig, SpanAttrs, ViemClient, DryrunResult, BundledOrders } from "../types"; /** * Executes a extimateGas call for an inter-orderbook arb() tx, to determine if the tx is successfull ot not @@ -20,6 +21,8 @@ export async function dryrun({ outputToEthPrice, config, viemClient, + l1Signer, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -31,6 +34,8 @@ export async function dryrun({ outputToEthPrice: string; opposingOrders: BundledOrders; maximumInput: BigNumber; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -105,13 +110,13 @@ export async function dryrun({ // trying to find opp with doing gas estimation, once to get gas and calculate // minimum sender output and second time to check the arb() with headroom - let gasLimit, blockNumber; + let gasLimit, blockNumber, l1Cost; try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) - .mul(config.gasLimitMultiplier) - .div(100); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + l1Cost = estimation.l1Cost; + gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); } catch (e) { const isNodeError = containsNodeError(e as BaseError); const errMsg = errorSnapshot("", e); @@ -133,7 +138,7 @@ export async function dryrun({ } return Promise.reject(result); } - let gasCost = gasLimit.mul(gasPrice); + let gasCost = gasLimit.mul(gasPrice).add(l1Cost); // repeat the same process with heaedroom if gas // coverage is not 0, 0 gas coverage means 0 minimum @@ -152,13 +157,13 @@ export async function dryrun({ ]); try { - blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); rawtx.gas = gasLimit.toBigInt(); - gasCost = gasLimit.mul(gasPrice); + gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); task.evaluable.bytecode = getBountyEnsureBytecode( ethers.utils.parseUnits(inputToEthPrice), ethers.utils.parseUnits(outputToEthPrice), @@ -229,6 +234,8 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1Signer, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -239,6 +246,8 @@ export async function findOpp({ gasPrice: bigint; inputToEthPrice: string; outputToEthPrice: string; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { if (!arb) throw undefined; const spanAttributes: SpanAttrs = {}; @@ -297,6 +306,8 @@ export async function findOpp({ outputToEthPrice, config, viemClient, + l1Signer, + l1GasPrice, }); }), ); @@ -366,6 +377,8 @@ export async function binarySearch({ outputToEthPrice, config, viemClient, + l1Signer, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -377,6 +390,8 @@ export async function binarySearch({ outputToEthPrice: string; opposingOrders: BundledOrders; maximumInput: ethers.BigNumber; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { const spanAttributes = {}; const result: DryrunResult = { @@ -400,6 +415,8 @@ export async function binarySearch({ outputToEthPrice, config, viemClient, + l1Signer, + l1GasPrice, }), ); // set the maxInput for next hop by increasing diff --git a/src/modes/intraOrderbook.ts b/src/modes/intraOrderbook.ts index acf56665..6314f1a8 100644 --- a/src/modes/intraOrderbook.ts +++ b/src/modes/intraOrderbook.ts @@ -1,3 +1,4 @@ +import { estimateGasCost } from "../gas"; import { BigNumber, ethers } from "ethers"; import { BaseError, PublicClient } from "viem"; import { erc20Abi, orderbookAbi } from "../abis"; @@ -5,12 +6,12 @@ import { getWithdrawEnsureBytecode } from "../config"; import { containsNodeError, errorSnapshot } from "../error"; import { estimateProfit, withBigintSerializer } from "../utils"; import { + SpanAttrs, BotConfig, - BundledOrders, ViemClient, - TakeOrderDetails, DryrunResult, - SpanAttrs, + BundledOrders, + TakeOrderDetails, } from "../types"; /** @@ -27,6 +28,8 @@ export async function dryrun({ viemClient, inputBalance, outputBalance, + l1Signer, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -38,6 +41,8 @@ export async function dryrun({ inputBalance: BigNumber; outputBalance: BigNumber; opposingOrder: TakeOrderDetails; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -102,13 +107,13 @@ export async function dryrun({ // trying to find opp with doing gas estimation, once to get gas and calculate // minimum sender output and second time to check the clear2() with withdraw2() and headroom - let gasLimit, blockNumber; + let gasLimit, blockNumber, l1Cost; try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) - .mul(config.gasLimitMultiplier) - .div(100); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + l1Cost = estimation.l1Cost; + gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); } catch (e) { // reason, code, method, transaction, error, stack, message const isNodeError = containsNodeError(e as BaseError); @@ -131,7 +136,7 @@ export async function dryrun({ } return Promise.reject(result); } - let gasCost = gasLimit.mul(gasPrice); + let gasCost = gasLimit.mul(gasPrice).add(l1Cost); // repeat the same process with heaedroom if gas // coverage is not 0, 0 gas coverage means 0 minimum @@ -159,13 +164,13 @@ export async function dryrun({ ]); try { - blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); rawtx.gas = gasLimit.toBigInt(); - gasCost = gasLimit.mul(gasPrice); + gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); task.evaluable.bytecode = getWithdrawEnsureBytecode( signer.account.address, orderPairObject.buyToken, @@ -241,6 +246,8 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1Signer, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -250,6 +257,8 @@ export async function findOpp({ gasPrice: bigint; inputToEthPrice: string; outputToEthPrice: string; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -320,6 +329,8 @@ export async function findOpp({ viemClient, inputBalance, outputBalance, + l1Signer, + l1GasPrice, }); } catch (e: any) { allNoneNodeErrors.push(e?.value?.noneNodeError); diff --git a/src/modes/routeProcessor.ts b/src/modes/routeProcessor.ts index 20cd8935..b961123f 100644 --- a/src/modes/routeProcessor.ts +++ b/src/modes/routeProcessor.ts @@ -1,10 +1,11 @@ import { Token } from "sushi/currency"; +import { estimateGasCost } from "../gas"; import { BaseError, PublicClient } from "viem"; import { getBountyEnsureBytecode } from "../config"; import { ChainId, DataFetcher, Router } from "sushi"; import { BigNumber, Contract, ethers } from "ethers"; import { containsNodeError, errorSnapshot } from "../error"; -import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; +import { SpanAttrs, BotConfig, ViemClient, DryrunResult, BundledOrders } from "../types"; import { scale18, scale18To, @@ -49,6 +50,8 @@ export async function dryrun({ config, viemClient, hasPriceMatch, + l1Signer, + l1GasPrice, }: { mode: number; config: BotConfig; @@ -63,6 +66,8 @@ export async function dryrun({ fromToken: Token; maximumInput: BigNumber; hasPriceMatch?: { value: boolean }; + l1Signer?: any; + l1GasPrice?: bigint; }) { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -180,11 +185,13 @@ export async function dryrun({ // trying to find opp with doing gas estimation, once to get gas and calculate // minimum sender output and second to check the arb() with headroom - let gasLimit, blockNumber; + let gasLimit, blockNumber, l1Cost; try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + l1Cost = estimation.l1Cost; + gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); } catch (e) { @@ -210,7 +217,7 @@ export async function dryrun({ } return Promise.reject(result); } - let gasCost = gasLimit.mul(gasPrice); + let gasCost = gasLimit.mul(gasPrice).add(l1Cost); // repeat the same process with heaedroom if gas // coverage is not 0, 0 gas coverage means 0 minimum @@ -229,13 +236,19 @@ export async function dryrun({ ]); try { - blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - gasLimit = ethers.BigNumber.from(await signer.estimateGas(rawtx)) + const estimation = await estimateGasCost( + rawtx, + signer, + config, + l1GasPrice, + l1Signer, + ); + gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); rawtx.gas = gasLimit.toBigInt(); - gasCost = gasLimit.mul(gasPrice); + gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); task.evaluable.bytecode = getBountyEnsureBytecode( ethers.utils.parseUnits(ethPrice), ethers.constants.Zero, @@ -315,6 +328,8 @@ export async function findOpp({ ethPrice, config, viemClient, + l1GasPrice, + l1Signer, }: { mode: number; config: BotConfig; @@ -327,6 +342,8 @@ export async function findOpp({ ethPrice: string; toToken: Token; fromToken: Token; + l1GasPrice?: bigint; + l1Signer?: any; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -362,6 +379,8 @@ export async function findOpp({ config, viemClient, hasPriceMatch, + l1GasPrice, + l1Signer, }); } catch (e: any) { // the fail reason can only be no route in case all hops fail reasons are no route @@ -394,6 +413,8 @@ export async function findOpp({ ethPrice, config, viemClient, + l1GasPrice, + l1Signer, }); } catch (e: any) { // the fail reason can only be no route in case all hops fail reasons are no route @@ -436,6 +457,8 @@ export async function findOppWithRetries({ ethPrice, config, viemClient, + l1GasPrice, + l1Signer, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -447,6 +470,8 @@ export async function findOppWithRetries({ ethPrice: string; toToken: Token; fromToken: Token; + l1Signer?: any; + l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -470,6 +495,8 @@ export async function findOppWithRetries({ ethPrice, config, viemClient, + l1GasPrice, + l1Signer, }), ); } diff --git a/src/processOrders.ts b/src/processOrders.ts index 33849abc..ce64db2c 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -34,6 +34,7 @@ import { getActualClearAmount, withBigintSerializer, } from "./utils"; +import { getL1Fee } from "./gas"; /** * Specifies reason that order process halted @@ -696,7 +697,10 @@ export async function processPair(args: { timeout: 200_000, }); - const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul(receipt.gasUsed); + const l1Fee = getL1Fee(receipt, config); + const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice) + .mul(receipt.gasUsed) + .add(l1Fee); signer.BALANCE = signer.BALANCE.sub(actualGasCost); if (receipt.status === "success") { spanAttributes["didClear"] = true; @@ -723,10 +727,13 @@ export async function processPair(args: { ); const netProfit = income ? income.sub(actualGasCost) : undefined; + spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); + if (config.isL2 && l1Fee) { + spanAttributes["details.gasCostL1"] = toNumber(l1Fee); + } if (income) { spanAttributes["details.income"] = toNumber(income); spanAttributes["details.netProfit"] = toNumber(netProfit!); - spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); } if (inputTokenIncome) { spanAttributes["details.inputTokenIncome"] = ethers.utils.formatUnits( @@ -801,10 +808,15 @@ export async function processPair(args: { // keep track of gas consumption of the account let actualGasCost; try { - actualGasCost = ethers.BigNumber.from(e.receipt.effectiveGasPrice).mul( - e.receipt.gasUsed, - ); + const l1Fee = getL1Fee(e.receipt, config); + actualGasCost = ethers.BigNumber.from(e.receipt.effectiveGasPrice) + .mul(e.receipt.gasUsed) + .add(l1Fee); signer.BALANCE = signer.BALANCE.sub(actualGasCost); + spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); + if (config.isL2 && l1Fee) { + spanAttributes["details.gasCostL1"] = toNumber(l1Fee); + } } catch { /**/ } diff --git a/src/types.ts b/src/types.ts index 0db97980..b0f5ab07 100644 --- a/src/types.ts +++ b/src/types.ts @@ -142,6 +142,7 @@ export type ChainConfig = { nativeWrappedToken: Token; routeProcessors: { [key: string]: `0x${string}` }; stableTokens?: Token[]; + isL2: boolean; }; export type BotConfig = { @@ -149,6 +150,7 @@ export type BotConfig = { nativeWrappedToken: Token; routeProcessors: { [key: string]: `0x${string}` }; stableTokens?: Token[]; + isL2: boolean; key?: string; mnemonic?: string; rpc: string[]; diff --git a/test/findOpp.test.js b/test/findOpp.test.js index 5d15c616..2c899f02 100644 --- a/test/findOpp.test.js +++ b/test/findOpp.test.js @@ -21,7 +21,7 @@ const oppBlockNumber = 123456; const { inputToEthPrice, outputToEthPrice, - gasPrice, + gasPrice: _gasPrice, gasLimitEstimation, arb, vaultBalance, @@ -39,6 +39,7 @@ const { orderbook, getAmountOut, } = testData; +const gasPrice = _gasPrice.toBigInt(); describe("Test find opp", async function () { beforeEach(() => { diff --git a/test/gas.test.ts b/test/gas.test.ts new file mode 100644 index 00000000..5b7c9164 --- /dev/null +++ b/test/gas.test.ts @@ -0,0 +1,61 @@ +import { assert } from "chai"; +import { estimateGasCost, getL1Fee } from "../src/gas"; +import { BotConfig, ViemClient } from "../src/types"; +import { createViemClient, getChainConfig } from "../src/config"; +import { ChainId } from "sushi"; + +describe("Test gas", async function () { + it("should estimate gas correctly for L1 and L2 chains", async function () { + // mock l1 signer + const l1Signer = { + getL1BaseFee: async () => 20n, + estimateL1Gas: async () => 5n, + }; + // mock normal signer + const signer = { + estimateGas: async () => 55n, + getGasPrice: async () => 2n, + } as any as ViemClient; + const tx = { + data: "0x1234" as `0x${string}`, + to: ("0x" + "1".repeat(40)) as `0x${string}`, + }; + + // estimate gas as L2 chain + const botconfig = { isL2: true } as any as BotConfig; + const result1 = await estimateGasCost(tx, signer, botconfig, undefined, l1Signer); + const expected1 = { + gas: 55n, + gasPrice: 2n, + l1Gas: 5n, + l1GasPrice: 20n, + l1Cost: 20n * 5n, + totalGasCost: 2n * 55n + 20n * 5n, + }; + assert.deepEqual(result1, expected1); + + // estimate as none L2 chain + botconfig.isL2 = false; + const result2 = await estimateGasCost(tx, signer, botconfig); + const expected2 = { + gas: 55n, + gasPrice: 2n, + l1Gas: 0n, + l1GasPrice: 0n, + l1Cost: 0n, + totalGasCost: 2n * 55n, + }; + assert.deepEqual(result2, expected2); + }); + + it("should get tx L1 gas cost from receipt", async function () { + const config = getChainConfig(ChainId.BASE); + const viemclient = await createViemClient(ChainId.BASE, ["https://rpc.ankr.com/base"]); + const hash = "0x18219497dc46babfbdc58fad112bf01ed584148bf06727cc97cb105915fd96b0"; + const receipt = await viemclient.getTransactionReceipt({ hash }); + + const result = getL1Fee(receipt, config as any as BotConfig); + const expected = 43615200401n; // known L1 cost taken from the actual tx + assert.equal(result, expected); + }); +}); diff --git a/test/processPair.test.js b/test/processPair.test.js index 9a22c100..cded1194 100644 --- a/test/processPair.test.js +++ b/test/processPair.test.js @@ -53,8 +53,8 @@ describe("Test process pair", async function () { BOUNTY: [], getAddress: () => "0x1F1E4c845183EF6d50E9609F16f6f9cAE43BC9Cb", getBlockNumber: async () => 123456, - getGasPrice: async () => gasPrice, - estimateGas: async () => gasLimitEstimation, + getGasPrice: async () => gasPrice.toBigInt(), + estimateGas: async () => gasLimitEstimation.toBigInt(), sendTransaction: async () => txHash, getTransactionCount: async () => 0, waitForTransactionReceipt: async () => { @@ -146,6 +146,7 @@ describe("Test process pair", async function () { "details.txUrl": scannerUrl + "/tx/" + txHash, "details.pair": pair, "details.gasPrice": gasPrice.mul(107).div(100).toString(), + "details.actualGasCost": Number(formatUnits(effectiveGasPrice.mul(gasUsed))), foundOpp: true, didClear: true, "details.inputToEthPrice": formatUnits(getCurrentInputToEthPrice()), @@ -221,6 +222,7 @@ describe("Test process pair", async function () { "details.txUrl": scannerUrl + "/tx/" + txHash, "details.pair": pair, "details.gasPrice": gasPrice.mul(107).div(100).toString(), + "details.actualGasCost": Number(formatUnits(effectiveGasPrice.mul(gasUsed))), foundOpp: true, didClear: true, "details.marketQuote.num": 0.99699, From 69cf8892ec804b16153f8ddf974d7e3ede1a7b9d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 2 Dec 2024 03:40:50 +0000 Subject: [PATCH 06/17] update --- src/config.ts | 8 +++--- src/gas.ts | 58 ++++++++++++++------------------------------ src/modes/index.ts | 13 +++------- src/processOrders.ts | 4 +-- src/types.ts | 4 +-- test/gas.test.ts | 8 +++--- 6 files changed, 34 insertions(+), 61 deletions(-) diff --git a/src/config.ts b/src/config.ts index c6cc304c..ebd5c47c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -58,7 +58,7 @@ export function getChainConfig(chainId: ChainId): ChainConfig { nativeWrappedToken, routeProcessors, stableTokens, - isL2: L2Chains.is(chain.id), + isSpecialL2: SpecialL2Chains.is(chain.id), }; } @@ -289,13 +289,13 @@ export function getWithdrawEnsureBytecode( * such as Arbitrum and Polygon zkEvm are excluded, these chains' * gas actions are performed the same as usual L1 chains. */ -export enum L2Chains { +export enum SpecialL2Chains { BASE = ChainId.BASE, OPTIMISM = ChainId.OPTIMISM, } -export namespace L2Chains { +export namespace SpecialL2Chains { export function is(chainId: number): boolean { - return Object.values(L2Chains).includes(chainId as any); + return Object.values(SpecialL2Chains).includes(chainId as any); } } diff --git a/src/gas.ts b/src/gas.ts index c65b40ad..81cfb58e 100644 --- a/src/gas.ts +++ b/src/gas.ts @@ -2,11 +2,7 @@ import { BotConfig, RawTx, ViemClient } from "./types"; import { publicActionsL2, walletActionsL2 } from "viem/op-stack"; /** - * Estimates gas cost of the given tx, also takes into account L1 gas cost if operating chain is L2. - * Not all L2 chains need to calculate L1 gas separately, some chains like Arbitrum and Polygon zkEvm, - * dont actually need anything extra other than usual gas estimation and they actually contain L1 gas in - * their usual gas estimation opertaion, but some other L2 chains such as Base and Optimism, do need to - * estimate L1 gas separately, so we will use a try/catch block + * Estimates gas cost of the given tx, also takes into account L1 gas cost if the chain is a special L2. */ export async function estimateGasCost( tx: RawTx, @@ -17,7 +13,15 @@ export async function estimateGasCost( ) { const gasPrice = tx.gasPrice ?? (await signer.getGasPrice()); const gas = await signer.estimateGas(tx); - if (config.isL2) { + const result = { + gas, + gasPrice, + l1Gas: 0n, + l1GasPrice: 0n, + l1Cost: 0n, + totalGasCost: gasPrice * gas, + }; + if (config.isSpecialL2) { try { const l1Signer_ = l1Signer ? l1Signer @@ -29,46 +33,20 @@ export async function estimateGasCost( to: tx.to, data: tx.data, }); - return { - gas, - gasPrice, - l1Gas, - l1GasPrice, - l1Cost: l1Gas * l1GasPrice, - totalGasCost: gasPrice * gas + l1Gas * l1GasPrice, - }; - } catch { - return { - gas, - gasPrice, - l1Gas: 0n, - l1GasPrice: 0n, - l1Cost: 0n, - totalGasCost: gasPrice * gas, - }; - } - } else { - return { - gas, - gasPrice, - l1Gas: 0n, - l1GasPrice: 0n, - l1Cost: 0n, - totalGasCost: gasPrice * gas, - }; + result.l1Gas = l1Gas; + result.l1GasPrice = l1GasPrice; + result.l1Cost = l1Gas * l1GasPrice; + result.totalGasCost += result.l1Cost; + } catch {} } + return result; } /** - * Retruns the L1 gas cost of a transaction if operating chain is L2 else returns 0. - * Not all L2 chains need report the L1 gas separately to the usual transaction receipt, chains - * like Arbitrum and Polygon zkEvm report the tx used gas normally like any other L1 chain, but - * some other L2 chains like Base and Optimism report the L1 gas separately to L2 using the properties - * below, so we need to explicitly check for the, if they are not present in the receipt, then simply - * return 0 + * Retruns the L1 gas cost of a transaction if operating chain is special L2 else returns 0. */ export function getL1Fee(receipt: any, config: BotConfig): bigint { - if (!config.isL2) return 0n; + if (!config.isSpecialL2) return 0n; if (typeof receipt.l1Fee === "bigint") { return receipt.l1Fee; diff --git a/src/modes/index.ts b/src/modes/index.ts index 43dd8e63..a454e1c1 100644 --- a/src/modes/index.ts +++ b/src/modes/index.ts @@ -53,19 +53,14 @@ export async function findOpp({ /**/ } - // if chain is L2, get L1 gas price before dryruns + // if chain is special L2, get L1 gas price just before dryruns let l1Signer; - let l1GasPrice: bigint | undefined; - if (config.isL2) { + let l1GasPrice = 0n; + if (config.isSpecialL2) { try { - // as already known, not all L2 chains support this method such as Arbitrum, - // only certain ones do, such as Base and Optimism, so use try/catch block - // and set to 0 on catch block so it becomes ineffective l1Signer = signer.extend(walletActionsL2()).extend(publicActionsL2()); l1GasPrice = await l1Signer.getL1BaseFee(); - } catch { - l1GasPrice = 0n; - } + } catch {} } const promises = [ diff --git a/src/processOrders.ts b/src/processOrders.ts index ce64db2c..a4ec1b27 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -728,7 +728,7 @@ export async function processPair(args: { const netProfit = income ? income.sub(actualGasCost) : undefined; spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); - if (config.isL2 && l1Fee) { + if (config.isSpecialL2 && l1Fee) { spanAttributes["details.gasCostL1"] = toNumber(l1Fee); } if (income) { @@ -814,7 +814,7 @@ export async function processPair(args: { .add(l1Fee); signer.BALANCE = signer.BALANCE.sub(actualGasCost); spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); - if (config.isL2 && l1Fee) { + if (config.isSpecialL2 && l1Fee) { spanAttributes["details.gasCostL1"] = toNumber(l1Fee); } } catch { diff --git a/src/types.ts b/src/types.ts index b0f5ab07..d2067064 100644 --- a/src/types.ts +++ b/src/types.ts @@ -142,7 +142,7 @@ export type ChainConfig = { nativeWrappedToken: Token; routeProcessors: { [key: string]: `0x${string}` }; stableTokens?: Token[]; - isL2: boolean; + isSpecialL2: boolean; }; export type BotConfig = { @@ -150,7 +150,7 @@ export type BotConfig = { nativeWrappedToken: Token; routeProcessors: { [key: string]: `0x${string}` }; stableTokens?: Token[]; - isL2: boolean; + isSpecialL2: boolean; key?: string; mnemonic?: string; rpc: string[]; diff --git a/test/gas.test.ts b/test/gas.test.ts index 5b7c9164..c5dbf777 100644 --- a/test/gas.test.ts +++ b/test/gas.test.ts @@ -21,8 +21,8 @@ describe("Test gas", async function () { to: ("0x" + "1".repeat(40)) as `0x${string}`, }; - // estimate gas as L2 chain - const botconfig = { isL2: true } as any as BotConfig; + // estimate gas as special L2 chain + const botconfig = { isSpecialL2: true } as any as BotConfig; const result1 = await estimateGasCost(tx, signer, botconfig, undefined, l1Signer); const expected1 = { gas: 55n, @@ -34,8 +34,8 @@ describe("Test gas", async function () { }; assert.deepEqual(result1, expected1); - // estimate as none L2 chain - botconfig.isL2 = false; + // estimate as usual chain + botconfig.isSpecialL2 = false; const result2 = await estimateGasCost(tx, signer, botconfig); const expected2 = { gas: 55n, From a13a0e909f9cb5bd1f08435b9e8d29f273d473b1 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 3 Dec 2024 01:50:12 +0000 Subject: [PATCH 07/17] init --- README.md | 4 +++ example.env | 3 ++ package-lock.json | 8 ++--- package.json | 2 +- src/cli.ts | 15 ++++++++ src/index.ts | 1 + src/modes/interOrderbook.ts | 21 ++++++++--- src/modes/intraOrderbook.ts | 3 ++ src/processOrders.ts | 13 ++----- src/query.ts | 5 +-- src/types.ts | 2 ++ src/utils.ts | 4 +++ test/e2e/e2e.test.js | 71 +++++++++++++++++++------------------ 13 files changed, 94 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index ce610a66..f0b0118b 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ Other optional arguments are: - `--gas-price-multiplier`, Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7%. Will override the 'GAS_PRICE_MULTIPLIER' in env variables - `--gas-limit-multiplier`, Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables - `--tx-gas`, Option to set a static gas limit for all submitting txs. Will override the 'TX_GAS' in env variables +- `--quote-gas`, Option to set a static gas limit for quote read calls, default is 1 milion. Will override the 'QUOTE_GAS' in env variables - `-V` or `--version`, output the version number - `-h` or `--help`, output usage information @@ -267,6 +268,9 @@ GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs TX_GAS= + +# Option to set a static gas limit for quote read calls, default is 1 milion +QUOTE_GAS= ``` If both env variables and CLI argument are set, the CLI arguments will be prioritized and override the env variables. diff --git a/example.env b/example.env index 61d8b14a..5ae79ac6 100644 --- a/example.env +++ b/example.env @@ -94,6 +94,9 @@ GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs TX_GAS= +# Option to set a static gas limit for quote read calls, default is 1 milion +QUOTE_GAS= + # test rpcs vars TEST_POLYGON_RPC= diff --git a/package-lock.json b/package-lock.json index e083100b..fdddfcac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@opentelemetry/resources": "^1.22.0", "@opentelemetry/sdk-trace-base": "^1.22.0", "@opentelemetry/semantic-conventions": "^1.22.0", - "@rainlanguage/orderbook": "^0.0.1-alpha.1", + "@rainlanguage/orderbook": "^0.0.1-alpha.6", "axios": "^1.3.4", "commander": "^11.0.0", "dotenv": "^16.0.3", @@ -3461,9 +3461,9 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "node_modules/@rainlanguage/orderbook": { - "version": "0.0.1-alpha.1", - "resolved": "https://registry.npmjs.org/@rainlanguage/orderbook/-/orderbook-0.0.1-alpha.1.tgz", - "integrity": "sha512-t7SYpjASwZEyowXH1rVs6LQPwJGUe5rTHWuHQZJq6eJfjZeDwZi5plxtIAwnMeqKrVSMdbAvPZMn0MpfdpctAQ==", + "version": "0.0.1-alpha.6", + "resolved": "https://registry.npmjs.org/@rainlanguage/orderbook/-/orderbook-0.0.1-alpha.6.tgz", + "integrity": "sha512-qGPzlDh0ZMlkPeHS048k0VlhhVIhrB7Ea/UIAZENBai3AYA394Cl6MWu66JNdia9RhVTCApSqRQwmmwLFxb2DA==", "dependencies": { "buffer": "^6.0.3" }, diff --git a/package.json b/package.json index 43e665ec..39e70ab9 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "@opentelemetry/resources": "^1.22.0", "@opentelemetry/sdk-trace-base": "^1.22.0", "@opentelemetry/semantic-conventions": "^1.22.0", - "@rainlanguage/orderbook": "^0.0.1-alpha.1", + "@rainlanguage/orderbook": "^0.0.1-alpha.6", "axios": "^1.3.4", "commander": "^11.0.0", "dotenv": "^16.0.3", diff --git a/src/cli.ts b/src/cli.ts index 8e77a27c..5ea03498 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -71,6 +71,7 @@ const ENV_OPTIONS = { gasPriceMultiplier: process?.env?.GAS_PRICE_MULTIPLIER, gasLimitMultiplier: process?.env?.GAS_LIMIT_MULTIPLIER, txGas: process?.env?.TX_GAS, + quoteGas: process?.env?.QUOTE_GAS, route: process?.env?.ROUTE, ownerProfile: process?.env?.OWNER_PROFILE ? Array.from(process?.env?.OWNER_PROFILE.matchAll(/[^,\s]+/g)).map((v) => v[0]) @@ -200,6 +201,10 @@ const getOptions = async (argv: any, version?: string) => { "--tx-gas ", "Option to set a static gas limit for all submitting txs. Will override the 'TX_GAS' in env variables", ) + .option( + "--quote-gas ", + "Option to set a static gas limit for quote read calls, default is 1 milion. Will override the 'QUOTE_GAS' in env variables", + ) .description( [ "A NodeJS app to find and take arbitrage trades for Rain Orderbook orders against some DeFi liquidity providers, requires NodeJS v18 or higher.", @@ -242,6 +247,7 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.gasLimitMultiplier = cmdOptions.gasLimitMultiplier || getEnv(ENV_OPTIONS.gasLimitMultiplier); cmdOptions.txGas = cmdOptions.txGas || getEnv(ENV_OPTIONS.txGas); + cmdOptions.quoteGas = cmdOptions.quoteGas || getEnv(ENV_OPTIONS.quoteGas); cmdOptions.botMinBalance = cmdOptions.botMinBalance || getEnv(ENV_OPTIONS.botMinBalance); cmdOptions.ownerProfile = cmdOptions.ownerProfile || getEnv(ENV_OPTIONS.ownerProfile); cmdOptions.route = cmdOptions.route || getEnv(ENV_OPTIONS.route); @@ -459,6 +465,15 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? throw "invalid txGas value, must be an integer greater than zero"; } else throw "invalid txGas value, must be an integer greater than zero"; } + if (options.quoteGas) { + try { + options.quoteGas = BigInt(options.quoteGas); + } catch { + throw "invalid quoteGas value, must be an integer greater than equal zero"; + } + } else { + options.quoteGas = 1_000_000n; // default + } const poolUpdateInterval = _poolUpdateInterval * 60 * 1000; let ordersDetails: SgOrder[] = []; if (!process?.env?.CLI_STARTUP_TEST) { diff --git a/src/index.ts b/src/index.ts index b533d20d..cbdcef89 100644 --- a/src/index.ts +++ b/src/index.ts @@ -209,6 +209,7 @@ export async function getConfig( config.gasPriceMultiplier = options.gasPriceMultiplier; config.gasLimitMultiplier = options.gasLimitMultiplier; config.txGas = options.txGas; + config.quoteGas = options.quoteGas; // init accounts const { mainAccount, accounts } = await initAccounts(walletKey, config, options, tracer, ctx); diff --git a/src/modes/interOrderbook.ts b/src/modes/interOrderbook.ts index 8132d278..d272791d 100644 --- a/src/modes/interOrderbook.ts +++ b/src/modes/interOrderbook.ts @@ -292,12 +292,23 @@ export async function findOpp({ // filter out the same owner orders const opposingOrders = { ...v, - takeOrders: v.takeOrders.filter( - (e) => - e.takeOrder.order.owner.toLowerCase() !== - orderPairObject.takeOrders[0].takeOrder.order.owner.toLowerCase(), - ), + takeOrders: v.takeOrders + .filter( + (e) => + e.takeOrder.order.owner.toLowerCase() !== + orderPairObject.takeOrders[0].takeOrder.order.owner.toLowerCase() && + e.quote && + e.quote.maxOutput.gt(0), + ) + .sort((a, b) => + a.quote!.ratio.lt(b.quote!.ratio) + ? -1 + : a.quote!.ratio.gt(b.quote!.ratio) + ? 1 + : 0, + ), }; + if (!opposingOrders.takeOrders.length) throw ""; return dryrun({ orderPairObject, opposingOrders, diff --git a/src/modes/intraOrderbook.ts b/src/modes/intraOrderbook.ts index 4021b6b8..7d9441b1 100644 --- a/src/modes/intraOrderbook.ts +++ b/src/modes/intraOrderbook.ts @@ -293,6 +293,9 @@ export async function findOpp({ orderPairObject.takeOrders[0].takeOrder.order.owner.toLowerCase() && // only orders that (priceA x priceB < 1) can be profitbale v.quote!.ratio.mul(orderPairObject.takeOrders[0].quote!.ratio).div(ONE).lt(ONE), + ) + .sort((a, b) => + a.quote!.ratio.lt(b.quote!.ratio) ? -1 : a.quote!.ratio.gt(b.quote!.ratio) ? 1 : 0, ); if (!opposingOrders || !opposingOrders.length) throw undefined; diff --git a/src/processOrders.ts b/src/processOrders.ts index a4ec1b27..ffb3f53f 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -24,7 +24,6 @@ import { toNumber, getIncome, getEthPrice, - quoteOrders, routeExists, PoolBlackList, getMarketQuote, @@ -151,16 +150,6 @@ export const processOrders = async ( span.end(); }); - // batch quote orders to establish the orders to loop over - try { - await quoteOrders( - bundledOrders, - (config as any).isTest ? (config as any).quoteRpc : config.rpc, - ); - } catch (e) { - throw errorSnapshot("Failed to batch quote orders", e); - } - let avgGasCost: BigNumber | undefined; const reports: Report[] = []; for (const orderbookOrders of bundledOrders) { @@ -460,6 +449,8 @@ export async function processPair(args: { await quoteSingleOrder( orderPairObject, (config as any).isTest ? (config as any).quoteRpc : config.rpc, + undefined, + config.quoteGas, ); if (orderPairObject.takeOrders[0].quote?.maxOutput.isZero()) { result.report = { diff --git a/src/query.ts b/src/query.ts index 19f43c32..f1913d8a 100644 --- a/src/query.ts +++ b/src/query.ts @@ -257,6 +257,7 @@ export async function getOrderChanges( timeout?: number, span?: Span, ) { + timeout; let skip_ = skip; let count = 0; const allResults: SgTx[] = []; @@ -267,7 +268,7 @@ export async function getOrderChanges( const res = await axios.post( subgraph, { query: getTxsQuery(startTimestamp, skip_) }, - { headers: { "Content-Type": "application/json" }, timeout }, + { headers: { "Content-Type": "application/json" } }, ); if (typeof res?.data?.data?.transactions !== "undefined") { const txs = res.data.data.transactions; @@ -282,7 +283,7 @@ export async function getOrderChanges( break; } } catch (error) { - span?.addEvent(errorSnapshot(`Failed to get order changes ${subgraph}`, error)); + span?.addEvent(errorSnapshot(`Failed to get orders changes ${subgraph}`, error)); throw error; } } diff --git a/src/types.ts b/src/types.ts index d2067064..a8fec171 100644 --- a/src/types.ts +++ b/src/types.ts @@ -49,6 +49,7 @@ export type CliOptions = { gasPriceMultiplier: number; gasLimitMultiplier: number; txGas?: bigint; + quoteGas: bigint; }; export type TokenDetails = { @@ -176,6 +177,7 @@ export type BotConfig = { gasPriceMultiplier: number; gasLimitMultiplier: number; txGas?: bigint; + quoteGas: bigint; onFetchRequest?: (request: Request) => void; onFetchResponse?: (request: Response) => void; }; diff --git a/src/utils.ts b/src/utils.ts index e55991f7..cb3d7156 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -744,6 +744,7 @@ export async function quoteOrders( orderDetails: BundledOrders[][], rpcs: string[], blockNumber?: bigint, + gas?: bigint, multicallAddressOverride?: string, ): Promise { let quoteResults: any[] = []; @@ -762,6 +763,7 @@ export async function quoteOrders( targets, rpc, blockNumber, + gas, multicallAddressOverride, ); break; @@ -819,6 +821,7 @@ export async function quoteSingleOrder( orderDetails: BundledOrders, rpcs: string[], blockNumber?: bigint, + gas?: bigint, multicallAddressOverride?: string, ) { for (let i = 0; i < rpcs.length; i++) { @@ -834,6 +837,7 @@ export async function quoteSingleOrder( ] as any as QuoteTarget[], rpc, blockNumber, + gas, multicallAddressOverride, ) )[0]; diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index c67d3b7d..bfb4f65d 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -527,27 +527,6 @@ for (let i = 0; i < testData.length; i++) { } // mock quote responses - await mockServer - .forPost("/rpc") - .once() - .thenSendJsonRpcResult( - encodeQuoteResponse([ - ...tokens.slice(1).map((v) => [ - true, // success - v.depositAmount.mul("1" + "0".repeat(18 - v.decimals)), //maxout - ethers.constants.Zero, // ratio - ]), - ...tokens - .slice(1) - .map(() => [ - true, - tokens[0].depositAmount.mul( - "1" + "0".repeat(18 - tokens[0].decimals), - ), - ethers.constants.Zero, - ]), - ]), - ); for (let i = 1; i < tokens.length; i++) { const output = tokens[i].depositAmount.mul( "1" + "0".repeat(18 - tokens[i].decimals), @@ -590,6 +569,24 @@ for (let i = 0; i < testData.length; i++) { await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); + + // mock init quotes + orders.forEach((ob) => { + ob.forEach((pair) => { + pair.takeOrders.forEach((takeOrder) => { + takeOrder.quote = { + ratio: ethers.constants.Zero, + maxOutput: tokens + .find( + (t) => + t.contract.address.toLowerCase() === + pair.sellToken.toLowerCase(), + ) + ?.depositAmount.mul("1" + "0".repeat(18 - ob.decimals)), + }; + }); + }); + }); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders @@ -888,20 +885,6 @@ for (let i = 0; i < testData.length; i++) { for (let i = 0; i < tokens.length - 1; i++) { t0.push(tokens[0]); } - await mockServer - .forPost("/rpc") - .once() - .thenSendJsonRpcResult( - encodeQuoteResponse([ - ...[tokens[1], ...t0, ...tokens.slice(2)].flatMap((v) => [ - [ - true, // success - v.depositAmount.mul("1" + "0".repeat(18 - v.decimals)), //maxout - ethers.constants.Zero, // ratio - ], - ]), - ]), - ); for (let i = 1; i < tokens.length; i++) { const output = tokens[i].depositAmount.mul( "1" + "0".repeat(18 - tokens[i].decimals), @@ -943,6 +926,24 @@ for (let i = 0; i < testData.length; i++) { await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); + + // mock init quotes + orders.forEach((ob) => { + ob.forEach((pair) => { + pair.takeOrders.forEach((takeOrder) => { + takeOrder.quote = { + ratio: ethers.constants.Zero, + maxOutput: tokens + .find( + (t) => + t.contract.address.toLowerCase() === + pair.sellToken.toLowerCase(), + ) + ?.depositAmount.mul("1" + "0".repeat(18 - ob.decimals)), + }; + }); + }); + }); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders From cb534fc2b480c3f47cd0f05f5d9904a0fc4974a5 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 3 Dec 2024 01:54:31 +0000 Subject: [PATCH 08/17] update tests --- test/cli.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cli.test.js b/test/cli.test.js index 228c15ac..0b791cff 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -189,6 +189,8 @@ describe("Test cli", async function () { "110", "--tx-gas", "123456789", + "--quote-gas", + "7777", ]); const expected = { roundGap: 10000, @@ -210,12 +212,14 @@ describe("Test cli", async function () { gasPriceMultiplier: 120, gasLimitMultiplier: 110, txGas: 123456789n, + quoteGas: 7777n, }, options: { botMinBalance: "0.123", gasPriceMultiplier: 120, gasLimitMultiplier: 110, txGas: 123456789n, + quoteGas: 7777n, }, }; await sleep(1000); @@ -233,5 +237,7 @@ describe("Test cli", async function () { assert.equal(result.config.gasLimitMultiplier, expected.config.gasLimitMultiplier); assert.equal(result.options.txGas, expected.options.txGas); assert.equal(result.config.txGas, expected.config.txGas); + assert.equal(result.options.quoteGas, expected.options.quoteGas); + assert.equal(result.config.quoteGas, expected.config.quoteGas); }); }); From 201c3a44ff501ce70274da3c03e4c8574f24b3a4 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 5 Dec 2024 03:14:12 +0000 Subject: [PATCH 09/17] lint --- src/processOrders.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index 5605fd30..d9268bd6 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -815,7 +815,7 @@ export async function handleReceipt( const l1Fee = getL1Fee(receipt, config); const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice) .mul(receipt.gasUsed) - .add(l1Fee);; + .add(l1Fee); const signerBalance = signer.BALANCE; signer.BALANCE = signer.BALANCE.sub(actualGasCost); if (receipt.status === "success") { From 2f42342e1b8c3416c42ec6f793c292af62c1646b Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Wed, 11 Dec 2024 02:14:31 +0000 Subject: [PATCH 10/17] update --- src/processOrders.ts | 1 - src/tx.ts | 20 +++++++++++++++++--- src/utils.ts | 4 ++-- test/tx.test.ts | 10 +++++++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index 9c828bae..30d4e90d 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -31,7 +31,6 @@ import { checkOwnedOrders, quoteSingleOrder, } from "./utils"; -import { getL1Fee } from "./gas"; /** * Specifies reason that order process halted diff --git a/src/tx.ts b/src/tx.ts index af73f053..42a436a9 100644 --- a/src/tx.ts +++ b/src/tx.ts @@ -1,3 +1,4 @@ +import { getL1Fee } from "./gas"; import { Token } from "sushi/currency"; import { addWatchedToken } from "./account"; import { BigNumber, Contract, ethers } from "ethers"; @@ -145,8 +146,15 @@ export async function handleTransaction( // keep track of gas consumption of the account let actualGasCost; try { - actualGasCost = BigNumber.from(e.receipt.effectiveGasPrice).mul(e.receipt.gasUsed); + const l1Fee = getL1Fee(e.receipt, config); + actualGasCost = BigNumber.from(e.receipt.effectiveGasPrice) + .mul(e.receipt.gasUsed) + .add(l1Fee); signer.BALANCE = signer.BALANCE.sub(actualGasCost); + spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); + if (config.isSpecialL2 && l1Fee) { + spanAttributes["details.gasCostL1"] = toNumber(l1Fee); + } } catch { /**/ } @@ -196,7 +204,10 @@ export async function handleReceipt( config: BotConfig, time: number, ): Promise { - const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul(receipt.gasUsed); + const l1Fee = getL1Fee(receipt, config); + const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice) + .mul(receipt.gasUsed) + .add(l1Fee); const signerBalance = signer.BALANCE; signer.BALANCE = signer.BALANCE.sub(actualGasCost); @@ -224,10 +235,13 @@ export async function handleReceipt( ); const netProfit = income ? income.sub(actualGasCost) : undefined; + spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); + if (config.isSpecialL2 && l1Fee) { + spanAttributes["details.gasCostL1"] = toNumber(l1Fee); + } if (income) { spanAttributes["details.income"] = toNumber(income); spanAttributes["details.netProfit"] = toNumber(netProfit!); - spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); } if (inputTokenIncome) { spanAttributes["details.inputTokenIncome"] = ethers.utils.formatUnits( diff --git a/src/utils.ts b/src/utils.ts index e55991f7..a8035561 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1267,8 +1267,8 @@ export async function checkOwnedOrders( * Converts to a float number */ export function toNumber(value: BigNumberish): number { - const valueString = ethers.utils.formatUnits(value); - return Number.parseFloat(valueString.substring(0, valueString.includes(".") ? 18 : 17)); + // valueString.substring(0, valueString.includes(".") ? 18 : 17) + return Number.parseFloat(ethers.utils.formatUnits(value)); } /** diff --git a/test/tx.test.ts b/test/tx.test.ts index 1432e166..e25e44b6 100644 --- a/test/tx.test.ts +++ b/test/tx.test.ts @@ -107,6 +107,9 @@ describe("Test tx", async function () { spanAttributes: { "details.txUrl": scannerUrl + "/tx/" + txHash, didClear: true, + "details.actualGasCost": Number( + ethers.utils.formatUnits(effectiveGasPrice * gasUsed), + ), }, report: { status: ProcessPairReportStatus.FoundOpportunity, @@ -239,7 +242,12 @@ describe("Test tx", async function () { reason: undefined, error: undefined, gasCost: ethers.BigNumber.from(gasUsed * effectiveGasPrice), - spanAttributes: { didClear: true }, + spanAttributes: { + didClear: true, + "details.actualGasCost": Number( + ethers.utils.formatUnits(effectiveGasPrice * gasUsed), + ), + }, report: { status: 3, txUrl, From 2f17d11d576c7180c2f9cd1ffaf0e845b09052c8 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 12 Dec 2024 00:27:23 +0000 Subject: [PATCH 11/17] init [skip ci] --- src/processOrders.ts | 14 +++++++--- src/tx.ts | 57 +++++++++++++++------------------------- test/processPair.test.js | 1 + 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index 05bcacd2..cc6f02df 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -237,9 +237,6 @@ export const processOrders = async ( } for (const { settle, pair, orderPairObject } of results) { - // instantiate a span for this pair - const span = tracer.startSpan(`order_${pair}`, undefined, ctx); - span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); try { // settle the process results // this will return the report of the operation and in case @@ -247,6 +244,10 @@ export const processOrders = async ( // the root cause as well const result = await settle(); + // instantiate a span for this pair + const span = tracer.startSpan(`order_${pair}`, undefined, ctx); + span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); + // keep track of avg gas cost if (result.gasCost) { txGasCosts.push(result.gasCost); @@ -273,7 +274,12 @@ export const processOrders = async ( span.setAttribute("severity", ErrorSeverity.HIGH); span.setStatus({ code: SpanStatusCode.ERROR, message: "unexpected error" }); } + span.end(); } catch (e: any) { + // instantiate a span for this pair + const span = tracer.startSpan(`order_${pair}`, undefined, ctx); + span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); + // set the span attributes with the values gathered at processPair() span.setAttributes(e.spanAttributes); @@ -404,8 +410,8 @@ export const processOrders = async ( span.setAttribute("severity", ErrorSeverity.HIGH); span.setStatus({ code: SpanStatusCode.ERROR, message }); } + span.end(); } - span.end(); } return { reports, diff --git a/src/tx.ts b/src/tx.ts index af73f053..57a2cb14 100644 --- a/src/tx.ts +++ b/src/tx.ts @@ -85,11 +85,23 @@ export async function handleTransaction( } // start getting tx receipt in background and return the settler fn - const receiptPromise = viemClient.waitForTransactionReceipt({ - hash: txhash!, - confirmations: 1, - timeout: 120_000, - }); + const receiptPromise = (async () => { + try { + return await viemClient.waitForTransactionReceipt({ + hash: txhash!, + confirmations: 1, + timeout: 120_000, + }); + } catch { + // in case waiting for tx receipt was unsuccessful, try getting the receipt directly + try { + return await viemClient.getTransactionReceipt({ hash: txhash! }); + } catch { + await sleep(Math.max(90_000 + time - Date.now(), 0)); + return await viemClient.getTransactionReceipt({ hash: txhash! }); + } + } + })(); return async () => { try { const receipt = await receiptPromise; @@ -112,36 +124,6 @@ export async function handleTransaction( time, ); } catch (e: any) { - try { - const newReceipt = await (async () => { - try { - return await viemClient.getTransactionReceipt({ hash: txhash }); - } catch { - await sleep(Math.max(90_000 + time - Date.now(), 0)); - return await viemClient.getTransactionReceipt({ hash: txhash }); - } - })(); - if (newReceipt) { - return handleReceipt( - txhash, - newReceipt, - signer, - spanAttributes, - rawtx, - orderbook, - orderPairObject, - inputToEthPrice, - outputToEthPrice, - result, - txUrl, - pair, - toToken, - fromToken, - config, - time, - ); - } - } catch {} // keep track of gas consumption of the account let actualGasCost; try { @@ -292,7 +274,10 @@ export async function handleReceipt( ); if (result.snapshot.includes("simulation failed to find the revert reason")) { // wait at least 90s before simulating the revert tx - // in order for rpcs to catch up + // in order for rpcs to catch up, this is concurrent to + // whole bot operation, so ideally all of it or at least + // partially will overlap with when bot is processing other + // orders await sleep(Math.max(90_000 + time - Date.now(), 0)); return await handleRevert( signer, diff --git a/test/processPair.test.js b/test/processPair.test.js index 141778eb..952c46a9 100644 --- a/test/processPair.test.js +++ b/test/processPair.test.js @@ -684,6 +684,7 @@ describe("Test process pair", async function () { }; signer.sendTx = async () => txHash; viemClient.waitForTransactionReceipt = async () => Promise.reject(errorRejection); + viemClient.getTransactionReceipt = async () => Promise.reject(errorRejection); try { await ( await processPair({ From ca70a8d398d80ee99f6a87c49932d73d307df669 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 12 Dec 2024 03:33:06 +0000 Subject: [PATCH 12/17] replace json otel kv with scoped kv --- src/modes/index.ts | 15 +++- src/modes/interOrderbook.ts | 11 +-- src/modes/intraOrderbook.ts | 6 +- src/modes/routeProcessor.ts | 9 +-- src/utils.ts | 18 +++++ test/findOpp.test.js | 129 ++++++++++++++++++++++++++----- test/mode-interOrderbook.test.js | 17 ++-- test/mode-intraOrderbook.test.js | 14 ++-- test/mode-routeProcessor.test.js | 27 +++++-- test/utils.test.js | 31 +++++++- 10 files changed, 211 insertions(+), 66 deletions(-) diff --git a/src/modes/index.ts b/src/modes/index.ts index 80e08cdd..d0b741b7 100644 --- a/src/modes/index.ts +++ b/src/modes/index.ts @@ -6,6 +6,7 @@ import { findOpp as findInterObOpp } from "./interOrderbook"; import { findOpp as findIntraObOpp } from "./intraOrderbook"; import { findOppWithRetries as findRpOpp } from "./routeProcessor"; import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; +import { extendSpanAttributes } from "../utils"; /** * The main entrypoint for the main logic to find opps. @@ -119,18 +120,24 @@ export async function findOpp({ noneNodeError: undefined, }; if ((allResults[0] as any)?.reason?.spanAttributes) { - spanAttributes["route-processor"] = JSON.stringify( + extendSpanAttributes( + spanAttributes, (allResults[0] as any).reason.spanAttributes, + "routeProcessor", ); } if ((allResults[1] as any)?.reason?.spanAttributes) { - spanAttributes["intra-orderbook"] = JSON.stringify( - (allResults[1] as any).reason.spanAttributes["intraOrderbook"], + extendSpanAttributes( + spanAttributes, + (allResults[1] as any).reason.spanAttributes, + "intraOrderbook", ); } if ((allResults[2] as any)?.reason?.spanAttributes) { - spanAttributes["inter-orderbook"] = JSON.stringify( + extendSpanAttributes( + spanAttributes, (allResults[2] as any).reason.spanAttributes, + "interOrderbook", ); } if ((allResults[0] as any)?.reason?.value?.noneNodeError) { diff --git a/src/modes/interOrderbook.ts b/src/modes/interOrderbook.ts index fa1763d6..ec35a2e1 100644 --- a/src/modes/interOrderbook.ts +++ b/src/modes/interOrderbook.ts @@ -3,7 +3,7 @@ import { BaseError, PublicClient } from "viem"; import { getBountyEnsureBytecode } from "../config"; import { BigNumber, Contract, ethers } from "ethers"; import { containsNodeError, errorSnapshot } from "../error"; -import { estimateProfit, withBigintSerializer } from "../utils"; +import { estimateProfit, extendSpanAttributes, withBigintSerializer } from "../utils"; import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; /** @@ -338,12 +338,13 @@ export async function findOpp({ // } catch { // /**/ // } - const allOrderbooksAttributes: any = {}; for (let i = 0; i < e.errors.length; i++) { - allOrderbooksAttributes[opposingOrderbookOrders[i].orderbook] = - e.errors[i].spanAttributes; + extendSpanAttributes( + spanAttributes, + e.errors[i].spanAttributes, + "againstOrderbooks." + opposingOrderbookOrders[i].orderbook, + ); } - spanAttributes["againstOrderbooks"] = JSON.stringify(allOrderbooksAttributes); const noneNodeErrors = allNoneNodeErrors.filter((v) => !!v); if (allNoneNodeErrors.length && noneNodeErrors.length / allNoneNodeErrors.length > 0.5) { result.value = { diff --git a/src/modes/intraOrderbook.ts b/src/modes/intraOrderbook.ts index 9f69724a..7eedab02 100644 --- a/src/modes/intraOrderbook.ts +++ b/src/modes/intraOrderbook.ts @@ -3,7 +3,7 @@ import { BaseError, PublicClient } from "viem"; import { erc20Abi, orderbookAbi } from "../abis"; import { getWithdrawEnsureBytecode } from "../config"; import { containsNodeError, errorSnapshot } from "../error"; -import { estimateProfit, withBigintSerializer } from "../utils"; +import { estimateProfit, extendSpanAttributes, withBigintSerializer } from "../utils"; import { BotConfig, BundledOrders, @@ -287,7 +287,6 @@ export async function findOpp({ ); if (!opposingOrders || !opposingOrders.length) throw undefined; - const allErrorAttributes: string[] = []; const allNoneNodeErrors: (string | undefined)[] = []; const erc20 = new ethers.utils.Interface(erc20Abi); const inputBalance = ethers.BigNumber.from( @@ -326,10 +325,9 @@ export async function findOpp({ }); } catch (e: any) { allNoneNodeErrors.push(e?.value?.noneNodeError); - allErrorAttributes.push(JSON.stringify(e.spanAttributes)); + extendSpanAttributes(spanAttributes, e.spanAttributes, "intraOrderbook." + i); } } - spanAttributes["intraOrderbook"] = allErrorAttributes; const noneNodeErrors = allNoneNodeErrors.filter((v) => !!v); if (allNoneNodeErrors.length && noneNodeErrors.length / allNoneNodeErrors.length > 0.5) { result.value = { diff --git a/src/modes/routeProcessor.ts b/src/modes/routeProcessor.ts index de93eb3d..a85fae7b 100644 --- a/src/modes/routeProcessor.ts +++ b/src/modes/routeProcessor.ts @@ -12,6 +12,7 @@ import { estimateProfit, visualizeRoute, withBigintSerializer, + extendSpanAttributes, } from "../utils"; /** @@ -347,8 +348,6 @@ export async function findOpp({ ethers.constants.Zero, ); const maximumInput = BigNumber.from(initAmount.toString()); - - const allHopsAttributes: string[] = []; const allNoneNodeErrors: (string | undefined)[] = []; try { return await dryrun({ @@ -370,7 +369,7 @@ export async function findOpp({ // the fail reason can only be no route in case all hops fail reasons are no route if (e.reason !== RouteProcessorDryrunHaltReason.NoRoute) noRoute = false; allNoneNodeErrors.push(e?.value?.noneNodeError); - allHopsAttributes.push(JSON.stringify(e.spanAttributes)); + extendSpanAttributes(spanAttributes, e.spanAttributes, "full"); } if (!hasPriceMatch.value) { const maxTradeSize = findMaxInput({ @@ -403,12 +402,10 @@ export async function findOpp({ if (e.reason !== RouteProcessorDryrunHaltReason.NoRoute) noRoute = false; delete e.spanAttributes["rawtx"]; allNoneNodeErrors.push(e?.value?.noneNodeError); - allHopsAttributes.push(JSON.stringify(e.spanAttributes)); + extendSpanAttributes(spanAttributes, e.spanAttributes, "partial"); } } } - // in case of no successfull hop, allHopsAttributes will be included - spanAttributes["hops"] = allHopsAttributes; if (noRoute) result.reason = RouteProcessorDryrunHaltReason.NoRoute; else { diff --git a/src/utils.ts b/src/utils.ts index e55991f7..e03c2f92 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1391,3 +1391,21 @@ export function scale18To(value: BigNumberish, targetDecimals: BigNumberish): Bi return BigNumber.from(value).div("1" + "0".repeat(18 - decimals)); } } + +/** + * Adds the given k/v pairs to the spanAttributes by prepending the key with given header + */ +export function extendSpanAttributes( + spanAttributes: Record, + newAttributes: Record, + header: string, + excludeHeaderForKeys: string[] = [], +) { + for (const attrKey in newAttributes) { + if (!excludeHeaderForKeys.includes(attrKey)) { + spanAttributes[header + "." + attrKey] = newAttributes[attrKey]; + } else { + spanAttributes[attrKey] = newAttributes[attrKey]; + } + } +} diff --git a/test/findOpp.test.js b/test/findOpp.test.js index 45100f4d..f1f438e6 100644 --- a/test/findOpp.test.js +++ b/test/findOpp.test.js @@ -2,6 +2,7 @@ const { assert } = require("chai"); const testData = require("./data"); const { findOpp } = require("../src/modes"); const { orderbookAbi } = require("../src/abis"); +const { errorSnapshot } = require("../src/error"); const { clone, estimateProfit } = require("../src/utils"); const { ethers, @@ -346,31 +347,121 @@ describe("Test find opp", async function () { }); assert.fail("expected to reject, but resolved"); } catch (error) { + const expectedTakeOrdersConfigStruct = { + minimumInput: ethers.constants.One, + maximumInput: vaultBalance, + maximumIORatio: ethers.constants.MaxUint256, + orders: [orderPairObject.takeOrders[0].takeOrder], + data: expectedRouteData, + }; + const task = { + evaluable: { + interpreter: + orderPairObject.takeOrders[0].takeOrder.order.evaluable.interpreter, + store: orderPairObject.takeOrders[0].takeOrder.order.evaluable.store, + bytecode: getBountyEnsureBytecode( + ethers.utils.parseUnits(inputToEthPrice), + ethers.constants.Zero, + ethers.constants.Zero, + signer.account.address, + ), + }, + signedContext: [], + }; + const rawtx = JSON.stringify({ + data: arb.interface.encodeFunctionData("arb3", [ + orderPairObject.orderbook, + expectedTakeOrdersConfigStruct, + task, + ]), + to: arb.address, + gasPrice, + from: signer.account.address, + }); + const opposingMaxInput = vaultBalance + .mul(orderPairObject.takeOrders[0].quote.ratio) + .div(`1${"0".repeat(36 - orderPairObject.buyTokenDecimals)}`); + const opposingMaxIORatio = ethers.BigNumber.from(`1${"0".repeat(36)}`).div( + orderPairObject.takeOrders[0].quote.ratio, + ); + const obInterface = new ethers.utils.Interface(orderbookAbi); + const encodedFN = obInterface.encodeFunctionData("takeOrders2", [ + { + minimumInput: ethers.constants.One, + maximumInput: opposingMaxInput, + maximumIORatio: opposingMaxIORatio, + orders: opposingOrderPairObject.takeOrders.map((v) => v.takeOrder), + data: "0x", + }, + ]); + const expectedTakeOrdersConfigStruct2 = { + minimumInput: ethers.constants.One, + maximumInput: vaultBalance, + maximumIORatio: ethers.constants.MaxUint256, + orders: [orderPairObject.takeOrders[0].takeOrder], + data: ethers.utils.defaultAbiCoder.encode( + ["address", "address", "bytes"], + [ + opposingOrderPairObject.orderbook, + opposingOrderPairObject.orderbook, + encodedFN, + ], + ), + }; + const task2 = { + evaluable: { + interpreter: + orderPairObject.takeOrders[0].takeOrder.order.evaluable.interpreter, + store: orderPairObject.takeOrders[0].takeOrder.order.evaluable.store, + bytecode: getBountyEnsureBytecode( + ethers.utils.parseUnits(inputToEthPrice), + ethers.utils.parseUnits(outputToEthPrice), + ethers.constants.Zero, + signer.account.address, + ), + }, + signedContext: [], + }; + const rawtx2 = { + data: arb.interface.encodeFunctionData("arb3", [ + orderPairObject.orderbook, + expectedTakeOrdersConfigStruct2, + task2, + ]), + to: arb.address, + gasPrice, + from: signer.account.address, + }; const expected = { rawtx: undefined, oppBlockNumber: undefined, + noneNodeError: errorSnapshot("", err), spanAttributes: { - "route-processor": { - hops: [ - `{"amountIn":"${formatUnits(vaultBalance)}","marketPrice":"${formatUnits(getCurrentPrice(vaultBalance))}","route":${JSON.stringify(expectedRouteVisual)},"blockNumber":${oppBlockNumber},"error":"${ethers.errors.UNPREDICTABLE_GAS_LIMIT}"}`, - `{"amountIn":"${formatUnits(vaultBalance.div(2))}","marketPrice":"${formatUnits(getCurrentPrice(vaultBalance.div(2)))}","route":${JSON.stringify(expectedRouteVisual)},"blockNumber":${oppBlockNumber}}`, - `{"amountIn":"${formatUnits(vaultBalance.div(4))}","marketPrice":"${formatUnits(getCurrentPrice(vaultBalance.div(4)))}","route":${JSON.stringify(expectedRouteVisual)},"blockNumber":${oppBlockNumber}}`, - ], - }, - "inter-orderbook": { - againstOrderbooks: { - [opposingOrderbookAddress]: { - amountIn: formatUnits(vaultBalance), - amountOut: formatUnits(getAmountOut(vaultBalance), 6), - blockNumber: oppBlockNumber, - error: err, - }, - }, - }, + // rp span attrs + "routeProcessor.full.stage": 1, + "routeProcessor.full.rawtx": rawtx, + "routeProcessor.full.isNodeError": false, + "routeProcessor.full.route": expectedRouteVisual, + "routeProcessor.full.blockNumber": oppBlockNumber, + "routeProcessor.full.error": errorSnapshot("", err), + "routeProcessor.full.amountIn": formatUnits(vaultBalance), + "routeProcessor.full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), + "routeProcessor.full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), + + // inter-ob span attrs + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.stage`]: 1, + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.isNodeError`]: false, + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.blockNumber`]: + oppBlockNumber, + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.rawtx`]: + JSON.stringify(rawtx2), + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.maxInput`]: + vaultBalance.toString(), + [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.error`]: + errorSnapshot("", err), }, }; - assert.deepEqual(error.rawtx, expected.rawtx); - assert.deepEqual(error.oppBlockNumber, expected.oppBlockNumber); + assert.deepEqual(error, expected); } }); }); diff --git a/test/mode-interOrderbook.test.js b/test/mode-interOrderbook.test.js index 24ac4a2d..2239d2a9 100644 --- a/test/mode-interOrderbook.test.js +++ b/test/mode-interOrderbook.test.js @@ -338,16 +338,13 @@ describe("Test inter-orderbook find opp", async function () { }, reason: undefined, spanAttributes: { - againstOrderbooks: JSON.stringify({ - [opposingOrderbookAddress]: { - maxInput: vaultBalance.toString(), - blockNumber: oppBlockNumber, - stage: 1, - isNodeError: false, - error: errorSnapshot("", err), - rawtx: JSON.stringify(rawtx), - }, - }), + [`againstOrderbooks.${opposingOrderbookAddress}.blockNumber`]: oppBlockNumber, + [`againstOrderbooks.${opposingOrderbookAddress}.stage`]: 1, + [`againstOrderbooks.${opposingOrderbookAddress}.isNodeError`]: false, + [`againstOrderbooks.${opposingOrderbookAddress}.error`]: errorSnapshot("", err), + [`againstOrderbooks.${opposingOrderbookAddress}.rawtx`]: JSON.stringify(rawtx), + [`againstOrderbooks.${opposingOrderbookAddress}.maxInput`]: + vaultBalance.toString(), }, }; assert.deepEqual(error, expected); diff --git a/test/mode-intraOrderbook.test.js b/test/mode-intraOrderbook.test.js index 2eb00dbe..1a5b4376 100644 --- a/test/mode-intraOrderbook.test.js +++ b/test/mode-intraOrderbook.test.js @@ -335,15 +335,11 @@ describe("Test intra-orderbook find opp", async function () { }, reason: undefined, spanAttributes: { - intraOrderbook: [ - JSON.stringify({ - blockNumber: oppBlockNumber, - stage: 1, - isNodeError: false, - error: errorSnapshot("", err), - rawtx: JSON.stringify(rawtx), - }), - ], + "intraOrderbook.0.blockNumber": oppBlockNumber, + "intraOrderbook.0.stage": 1, + "intraOrderbook.0.isNodeError": false, + "intraOrderbook.0.error": errorSnapshot("", err), + "intraOrderbook.0.rawtx": JSON.stringify(rawtx), }, }; assert.deepEqual(error, expected); diff --git a/test/mode-routeProcessor.test.js b/test/mode-routeProcessor.test.js index 1dbdce30..276a21ca 100644 --- a/test/mode-routeProcessor.test.js +++ b/test/mode-routeProcessor.test.js @@ -476,9 +476,15 @@ describe("Test route processor find opp", async function () { }, reason: RouteProcessorDryrunHaltReason.NoOpportunity, spanAttributes: { - hops: [ - `{"amountIn":"${formatUnits(vaultBalance)}","amountOut":"${formatUnits(getAmountOut(vaultBalance), 6)}","marketPrice":"${formatUnits(getCurrentPrice(vaultBalance))}","route":${JSON.stringify(expectedRouteVisual)},"blockNumber":${oppBlockNumber},"stage":1,"isNodeError":false,"error":${JSON.stringify(errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT))},"rawtx":${JSON.stringify(rawtx)}}`, - ], + "full.amountIn": formatUnits(vaultBalance), + "full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), + "full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), + "full.route": expectedRouteVisual, + "full.blockNumber": oppBlockNumber, + "full.stage": 1, + "full.isNodeError": false, + "full.error": errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), + "full.rawtx": rawtx, }, }; assert.deepEqual(error, expected); @@ -509,7 +515,8 @@ describe("Test route processor find opp", async function () { value: undefined, reason: RouteProcessorDryrunHaltReason.NoRoute, spanAttributes: { - hops: [`{"amountIn":"${formatUnits(vaultBalance)}","route":"no-way"}`], + "full.amountIn": formatUnits(vaultBalance), + "full.route": "no-way", }, }; assert.deepEqual(error, expected); @@ -674,9 +681,15 @@ describe("Test find opp with retries", async function () { }, reason: RouteProcessorDryrunHaltReason.NoOpportunity, spanAttributes: { - hops: [ - `{"amountIn":"${formatUnits(vaultBalance)}","amountOut":"${formatUnits(getAmountOut(vaultBalance), 6)}","marketPrice":"${formatUnits(getCurrentPrice(vaultBalance))}","route":${JSON.stringify(expectedRouteVisual)},"blockNumber":${oppBlockNumber},"stage":1,"isNodeError":false,"error":${JSON.stringify(errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT))},"rawtx":${JSON.stringify(rawtx)}}`, - ], + "full.amountIn": formatUnits(vaultBalance), + "full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), + "full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), + "full.route": expectedRouteVisual, + "full.blockNumber": oppBlockNumber, + "full.stage": 1, + "full.isNodeError": false, + "full.error": errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), + "full.rawtx": rawtx, }, }; assert.deepEqual(error, expected); diff --git a/test/utils.test.js b/test/utils.test.js index 008d560b..458ab683 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -1,11 +1,18 @@ const { assert } = require("chai"); const testData = require("./data"); -const { clone, getTotalIncome, checkOwnedOrders, scale18, scale18To } = require("../src/utils"); const { ethers, - utils: { hexlify, randomBytes }, BigNumber, + utils: { hexlify, randomBytes }, } = require("ethers"); +const { + clone, + scale18, + scale18To, + getTotalIncome, + checkOwnedOrders, + extendSpanAttributes, +} = require("../src/utils"); describe("Test utils functions", async function () { it("should clone correctly", async function () { @@ -118,4 +125,24 @@ describe("Test utils functions", async function () { const expected2 = BigNumber.from("12345678900000"); assert.deepEqual(result2, expected2); }); + + it("should test extendSpanAttributes", async function () { + const newAttrs = { + a: 10, + b: true, + c: "some string", + }; + const spanAttrs = { + oldKey: "some value", + }; + extendSpanAttributes(spanAttrs, newAttrs, "header"); + + const expected = { + oldKey: "some value", + "header.a": 10, + "header.b": true, + "header.c": "some string", + }; + assert.deepEqual(spanAttrs, expected); + }); }); From d947f8b0d48bd93ca33c15662632055efefcf67b Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 19 Dec 2024 03:23:37 +0000 Subject: [PATCH 13/17] update --- src/tx.ts | 20 +------------------- src/utils.ts | 1 - 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/tx.ts b/src/tx.ts index 42a436a9..deb0cdc4 100644 --- a/src/tx.ts +++ b/src/tx.ts @@ -1,7 +1,7 @@ import { getL1Fee } from "./gas"; import { Token } from "sushi/currency"; +import { Contract, ethers } from "ethers"; import { addWatchedToken } from "./account"; -import { BigNumber, Contract, ethers } from "ethers"; import { containsNodeError, handleRevert } from "./error"; import { ProcessPairHaltReason, ProcessPairReportStatus } from "./processOrders"; import { BotConfig, BundledOrders, ProcessPairResult, RawTx, ViemClient } from "./types"; @@ -143,21 +143,6 @@ export async function handleTransaction( ); } } catch {} - // keep track of gas consumption of the account - let actualGasCost; - try { - const l1Fee = getL1Fee(e.receipt, config); - actualGasCost = BigNumber.from(e.receipt.effectiveGasPrice) - .mul(e.receipt.gasUsed) - .add(l1Fee); - signer.BALANCE = signer.BALANCE.sub(actualGasCost); - spanAttributes["details.actualGasCost"] = toNumber(actualGasCost); - if (config.isSpecialL2 && l1Fee) { - spanAttributes["details.gasCostL1"] = toNumber(l1Fee); - } - } catch { - /**/ - } result.report = { status: ProcessPairReportStatus.FoundOpportunity, txUrl, @@ -165,9 +150,6 @@ export async function handleTransaction( buyToken: orderPairObject.buyToken, sellToken: orderPairObject.sellToken, }; - if (actualGasCost) { - result.report.actualGasCost = ethers.utils.formatUnits(actualGasCost); - } result.error = e; spanAttributes["details.rawTx"] = JSON.stringify( { diff --git a/src/utils.ts b/src/utils.ts index a8035561..1a3464b2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1267,7 +1267,6 @@ export async function checkOwnedOrders( * Converts to a float number */ export function toNumber(value: BigNumberish): number { - // valueString.substring(0, valueString.includes(".") ? 18 : 17) return Number.parseFloat(ethers.utils.formatUnits(value)); } From e7facadf6e239be58423ae09d1e5ffd7cb9a0347 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 27 Dec 2024 04:00:46 +0000 Subject: [PATCH 14/17] update --- src/account.ts | 114 ++++++++++++++++-------------------- src/cli.ts | 25 ++++++-- src/gas.ts | 33 ++++++++++- src/index.ts | 13 +++- src/modes/index.ts | 28 ++------- src/modes/interOrderbook.ts | 12 +--- src/modes/intraOrderbook.ts | 9 +-- src/modes/routeProcessor.ts | 19 +----- src/processOrders.ts | 52 +++++++--------- src/tx.ts | 6 +- src/types.ts | 5 ++ test/account.test.js | 6 +- test/cli.test.js | 4 +- test/e2e/e2e.test.js | 15 ++++- test/gas.test.ts | 24 +++++++- test/processPair.test.js | 62 ++++---------------- 16 files changed, 204 insertions(+), 223 deletions(-) diff --git a/src/account.ts b/src/account.ts index 3117e0ae..2d6bc824 100644 --- a/src/account.ts +++ b/src/account.ts @@ -1,5 +1,6 @@ import { ChainId, RPParams } from "sushi"; import { BigNumber, ethers } from "ethers"; +import { estimateGasCost, getTxFee } from "./gas"; import { ErrorSeverity, errorSnapshot } from "./error"; import { Native, Token, WNATIVE } from "sushi/currency"; import { ROUTE_PROCESSOR_4_ADDRESS } from "sushi/config"; @@ -8,8 +9,15 @@ import { createViemClient, getDataFetcher } from "./config"; import { mnemonicToAccount, privateKeyToAccount } from "viem/accounts"; import { erc20Abi, multicall3Abi, orderbookAbi, routeProcessor3Abi } from "./abis"; import { context, Context, SpanStatusCode, trace, Tracer } from "@opentelemetry/api"; -import { BotConfig, CliOptions, ViemClient, TokenDetails, OwnedOrder } from "./types"; import { parseAbi, hexToNumber, numberToHex, PublicClient, NonceManagerSource } from "viem"; +import { + BotConfig, + CliOptions, + ViemClient, + OwnedOrder, + TokenDetails, + OperationState, +} from "./types"; /** Standard base path for eth accounts */ export const BasePath = "m/44'/60'/0'/0/" as const; @@ -112,9 +120,7 @@ export async function initAccounts( confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul( - receipt.gasUsed, - ); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); if (receipt.status === "success") { accounts[i].BALANCE = topupAmountBn; mainAccount.BALANCE = @@ -151,6 +157,7 @@ export async function manageAccounts( avgGasCost: BigNumber, lastIndex: number, wgc: ViemClient[], + state: OperationState, tracer?: Tracer, ctx?: Context, ) { @@ -159,11 +166,11 @@ export async function manageAccounts( for (let i = config.accounts.length - 1; i >= 0; i--) { if (config.accounts[i].BALANCE.lt(avgGasCost.mul(4))) { try { - const gasPrice = await config.viemClient.getGasPrice(); await sweepToMainWallet( config.accounts[i], config.mainAccount, - gasPrice, + state, + config, tracer, ctx, ); @@ -256,9 +263,7 @@ export async function manageAccounts( confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul( - receipt.gasUsed, - ); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); if (receipt.status === "success") { accountsToAdd--; acc.BALANCE = topupAmountBN; @@ -476,10 +481,12 @@ export async function getBatchTokenBalanceForAccount( export async function sweepToMainWallet( fromWallet: ViemClient, toWallet: ViemClient, - gasPrice: bigint, + state: OperationState, + config: BotConfig, tracer?: Tracer, ctx?: Context, ) { + const gasPrice = state.gasPrice; const mainSpan = tracer?.startSpan("sweep-wallet-funds", undefined, ctx); const mainCtx = mainSpan ? trace.setSpan(context.active(), mainSpan) : undefined; mainSpan?.setAttribute("details.wallet", fromWallet.account.address); @@ -488,7 +495,6 @@ export async function sweepToMainWallet( fromWallet.BOUNTY.map((v) => v.symbol), ); - gasPrice = ethers.BigNumber.from(gasPrice).mul(107).div(100).toBigInt(); const erc20 = new ethers.utils.Interface(erc20Abi); const txs: { bounty: TokenDetails; @@ -499,7 +505,7 @@ export async function sweepToMainWallet( }; }[] = []; const failedBounties: TokenDetails[] = []; - let cumulativeGasLimit = ethers.constants.Zero; + let cumulativeGas = ethers.constants.Zero; for (let i = 0; i < fromWallet.BOUNTY.length; i++) { const bounty = fromWallet.BOUNTY[i]; try { @@ -517,29 +523,28 @@ export async function sweepToMainWallet( continue; } const tx = { + gasPrice, to: bounty.address as `0x${string}`, data: erc20.encodeFunctionData("transfer", [ toWallet.account.address, balance, ]) as `0x${string}`, }; - const gas = await fromWallet.estimateGas(tx); + // const gas = await fromWallet.estimateGas(tx); + const gas = (await estimateGasCost(tx, fromWallet, config, state.l1GasPrice)) + .totalGasCost; txs.push({ tx, bounty, balance: ethers.utils.formatUnits(balance, bounty.decimals) }); - cumulativeGasLimit = cumulativeGasLimit.add(gas); + cumulativeGas = cumulativeGas.add(gas); } catch { addWatchedToken(bounty, failedBounties); } } - if (cumulativeGasLimit.mul(gasPrice).mul(125).div(100).gt(fromWallet.BALANCE)) { + if (cumulativeGas.mul(125).div(100).gt(fromWallet.BALANCE)) { const span = tracer?.startSpan("fund-wallet-to-sweep", undefined, mainCtx); span?.setAttribute("details.wallet", fromWallet.account.address); try { - const transferAmount = cumulativeGasLimit - .mul(gasPrice) - .mul(125) - .div(100) - .sub(fromWallet.BALANCE); + const transferAmount = cumulativeGas.mul(125).div(100).sub(fromWallet.BALANCE); span?.setAttribute("details.amount", ethers.utils.formatUnits(transferAmount)); const hash = await toWallet.sendTx({ to: fromWallet.account.address, @@ -550,7 +555,7 @@ export async function sweepToMainWallet( confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul(receipt.gasUsed); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); if (receipt.status === "success") { span?.setStatus({ code: SpanStatusCode.OK, @@ -592,7 +597,7 @@ export async function sweepToMainWallet( confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul(receipt.gasUsed); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); if (receipt.status === "success") { if (!toWallet.BOUNTY.find((v) => v.address === txs[i].bounty.address)) { toWallet.BOUNTY.push(txs[i].bounty); @@ -626,32 +631,31 @@ export async function sweepToMainWallet( const span = tracer?.startSpan("sweep-remaining-gas-to-main-wallet", undefined, mainCtx); span?.setAttribute("details.wallet", fromWallet.account.address); try { - const gasLimit = ethers.BigNumber.from( - await fromWallet.estimateGas({ - to: toWallet.account.address, - value: 0n, - }), + const estimation = await estimateGasCost( + { to: toWallet.account.address, value: 0n, gasPrice } as any, + fromWallet, + config, + state.l1GasPrice, ); + const remainingGas = ethers.BigNumber.from( await fromWallet.getBalance({ address: fromWallet.account.address }), ); - const transferAmount = remainingGas.sub(gasLimit.mul(gasPrice)); + const transferAmount = remainingGas.sub(estimation.totalGasCost); if (transferAmount.gt(0)) { span?.setAttribute("details.amount", ethers.utils.formatUnits(transferAmount)); const hash = await fromWallet.sendTx({ gasPrice, to: toWallet.account.address, value: transferAmount.toBigInt(), - gas: gasLimit.toBigInt(), + gas: estimation.gas, }); const receipt = await fromWallet.waitForTransactionReceipt({ hash, confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul( - receipt.gasUsed, - ); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); if (receipt.status === "success") { toWallet.BALANCE = toWallet.BALANCE.add(transferAmount); fromWallet.BALANCE = fromWallet.BALANCE.sub(txCost).sub(transferAmount); @@ -691,16 +695,19 @@ export async function sweepToMainWallet( * Sweep bot's bounties to eth * @param config - The config obj */ -export async function sweepToEth(config: BotConfig, tracer?: Tracer, ctx?: Context) { +export async function sweepToEth( + config: BotConfig, + state: OperationState, + tracer?: Tracer, + ctx?: Context, +) { const skipped: TokenDetails[] = []; const rp4Address = ROUTE_PROCESSOR_4_ADDRESS[ config.chain.id as keyof typeof ROUTE_PROCESSOR_4_ADDRESS ] as `0x${string}`; const rp = new ethers.utils.Interface(routeProcessor3Abi); const erc20 = new ethers.utils.Interface(erc20Abi); - const gasPrice = ethers.BigNumber.from(await config.mainAccount.getGasPrice()) - .mul(107) - .div(100); + const gasPrice = ethers.BigNumber.from(state.gasPrice); for (let i = 0; i < config.mainAccount.BOUNTY.length; i++) { const bounty = config.mainAccount.BOUNTY[i]; const span = tracer?.startSpan("sweep-to-gas", undefined, ctx); @@ -906,6 +913,7 @@ export function addWatchedToken( export async function fundOwnedOrders( ownedOrders: OwnedOrder[], config: BotConfig, + state: OperationState, ): Promise<{ ownedOrder?: OwnedOrder; error: string }[]> { const failedFundings: { ownedOrder?: OwnedOrder; error: string }[] = []; const ob = new ethers.utils.Interface(orderbookAbi); @@ -913,23 +921,7 @@ export async function fundOwnedOrders( const rp = new ethers.utils.Interface(routeProcessor3Abi); const rp4Address = ROUTE_PROCESSOR_4_ADDRESS[config.chain.id as keyof typeof ROUTE_PROCESSOR_4_ADDRESS]; - let gasPrice: BigNumber; - for (let i = 0; i < 4; i++) { - try { - gasPrice = ethers.BigNumber.from(await config.viemClient.getGasPrice()) - .mul(107) - .div(100); - break; - } catch (e) { - if (i == 3) - return [ - { - error: errorSnapshot("failed to get gas price", e), - }, - ]; - else await sleep(10000 * (i + 1)); - } - } + const gasPrice = ethers.BigNumber.from(state.gasPrice); if (config.selfFundOrders) { for (let i = 0; i < ownedOrders.length; i++) { const ownedOrder = ownedOrders[i]; @@ -973,7 +965,7 @@ export async function fundOwnedOrders( config.mainAccount.account.address, rp4Address, config.dataFetcher, - gasPrice!, + gasPrice, ); const initSellAmount = ethers.BigNumber.from(route.amountOutBI); let sellAmount: BigNumber; @@ -988,7 +980,7 @@ export async function fundOwnedOrders( config.mainAccount.account.address, rp4Address, config.dataFetcher, - gasPrice!, + gasPrice, ); if (topupAmount.lte(route.amountOutBI)) { finalRpParams = rpParams; @@ -1013,9 +1005,7 @@ export async function fundOwnedOrders( confirmations: 4, timeout: 100_000, }); - const swapTxCost = ethers.BigNumber.from( - swapReceipt.effectiveGasPrice, - ).mul(swapReceipt.gasUsed); + const swapTxCost = ethers.BigNumber.from(getTxFee(swapReceipt, config)); config.mainAccount.BALANCE = config.mainAccount.BALANCE.sub(swapTxCost); if (swapReceipt.status === "success") { config.mainAccount.BALANCE = config.mainAccount.BALANCE.sub( @@ -1050,8 +1040,8 @@ export async function fundOwnedOrders( timeout: 100_000, }); const approveTxCost = ethers.BigNumber.from( - approveReceipt.effectiveGasPrice, - ).mul(approveReceipt.gasUsed); + getTxFee(approveReceipt, config), + ); config.mainAccount.BALANCE = config.mainAccount.BALANCE.sub(approveTxCost); if (approveReceipt.status === "reverted") { @@ -1073,9 +1063,7 @@ export async function fundOwnedOrders( confirmations: 4, timeout: 100_000, }); - const txCost = ethers.BigNumber.from(receipt.effectiveGasPrice).mul( - receipt.gasUsed, - ); + const txCost = ethers.BigNumber.from(getTxFee(receipt, config)); config.mainAccount.BALANCE = config.mainAccount.BALANCE.sub(txCost); if (receipt.status === "success") { ownedOrder.vaultBalance = ownedOrder.vaultBalance.add(topupAmount); diff --git a/src/cli.ts b/src/cli.ts index b74d3d90..e312fe7c 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,5 @@ import { config } from "dotenv"; +import { getGasPrice } from "./gas"; import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; @@ -11,9 +12,9 @@ import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; -import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { BotConfig, BundledOrders, CliOptions, OperationState, ViemClient } from "./types"; import { sweepToEth, manageAccounts, @@ -303,6 +304,7 @@ export const arbRound = async ( options: CliOptions, config: BotConfig, bundledOrders: BundledOrders[][], + state: OperationState, ) => { return await tracer.startActiveSpan("process-orders", {}, roundCtx, async (span) => { const ctx = trace.setSpan(context.active(), span); @@ -314,6 +316,7 @@ export const arbRound = async ( const { reports = [], avgGasCost = undefined } = await clear( config, bundledOrders, + state, tracer, ctx, ); @@ -490,6 +493,13 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? ctx, ); + // fetch initial gas price on startup + const state: OperationState = { + gasPrice: 0n, + l1GasPrice: 0n, + }; + await getGasPrice(config, state); + return { roundGap, options: options as CliOptions, @@ -503,6 +513,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? ), tokens, lastReadOrdersTimestamp, + state, }; } @@ -549,6 +560,7 @@ export const main = async (argv: any, version?: string) => { orderbooksOwnersProfileMap, tokens, lastReadOrdersTimestamp, + state, } = await tracer.startActiveSpan("startup", async (startupSpan) => { const ctx = trace.setSpan(context.active(), startupSpan); try { @@ -571,6 +583,9 @@ export const main = async (argv: any, version?: string) => { } }); + // periodically fetch and set gas price in state (once every 20 seconds) + setInterval(() => getGasPrice(config, state), 20_000); + const lastReadOrdersMap = options.subgraph.map((v) => ({ sg: v, skip: 0, @@ -658,6 +673,7 @@ export const main = async (argv: any, version?: string) => { options, config, bundledOrders, + state, ); let txs, foundOpp, didClear, roundAvgGasCost; if (roundResult) { @@ -715,6 +731,7 @@ export const main = async (argv: any, version?: string) => { avgGasCost, lastUsedAccountIndex, wgc, + state, tracer, roundCtx, ); @@ -726,11 +743,11 @@ export const main = async (argv: any, version?: string) => { if (wgc.length) { for (let k = wgc.length - 1; k >= 0; k--) { try { - const gasPrice = await config.viemClient.getGasPrice(); await sweepToMainWallet( wgc[k], config.mainAccount, - gasPrice, + state, + config, tracer, roundCtx, ); @@ -765,7 +782,7 @@ export const main = async (argv: any, version?: string) => { } // try to sweep main wallet's tokens back to eth try { - await sweepToEth(config, tracer, roundCtx); + await sweepToEth(config, state, tracer, roundCtx); } catch { /**/ } diff --git a/src/gas.ts b/src/gas.ts index 81cfb58e..c8ef953b 100644 --- a/src/gas.ts +++ b/src/gas.ts @@ -1,5 +1,6 @@ -import { BotConfig, RawTx, ViemClient } from "./types"; +import { BigNumber } from "ethers"; import { publicActionsL2, walletActionsL2 } from "viem/op-stack"; +import { BotConfig, OperationState, RawTx, ViemClient } from "./types"; /** * Estimates gas cost of the given tx, also takes into account L1 gas cost if the chain is a special L2. @@ -11,7 +12,8 @@ export async function estimateGasCost( l1GasPrice?: bigint, l1Signer?: any, ) { - const gasPrice = tx.gasPrice ?? (await signer.getGasPrice()); + const gasPrice = + tx.gasPrice ?? ((await signer.getGasPrice()) * BigInt(config.gasPriceMultiplier)) / 100n; const gas = await signer.estimateGas(tx); const result = { gas, @@ -56,3 +58,30 @@ export function getL1Fee(receipt: any, config: BotConfig): bigint { return 0n; } } + +/** + * Get Transaction total gas cost from receipt (includes L1 fee) + */ +export function getTxFee(receipt: any, config: BotConfig): bigint { + const gasUsed = BigNumber.from(receipt.gasUsed).toBigInt(); + const effectiveGasPrice = BigNumber.from(receipt.effectiveGasPrice).toBigInt(); + return effectiveGasPrice * gasUsed + getL1Fee(receipt, config); +} + +/** + * Fetches the gas price (L1 gas price as well if chain is special L2) + */ +export async function getGasPrice(config: BotConfig, state: OperationState) { + const promises = [config.viemClient.getGasPrice()]; + if (config.isSpecialL2) { + const l1Client = config.viemClient.extend(publicActionsL2()); + promises.push(l1Client.getL1BaseFee()); + } + const [gasPriceResult, l1GasPriceResult = undefined] = await Promise.allSettled(promises); + if (gasPriceResult.status === "fulfilled") { + state.gasPrice = (gasPriceResult.value * BigInt(config.gasPriceMultiplier)) / 100n; + } + if (l1GasPriceResult?.status === "fulfilled") { + state.l1GasPrice = l1GasPriceResult.value; + } +} diff --git a/src/index.ts b/src/index.ts index 94e3579a..eae5838e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,15 @@ import { Context, Span } from "@opentelemetry/api"; import { checkSgStatus, handleSgResults } from "./sg"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { querySgOrders, SgOrder, statusCheckQuery } from "./query"; -import { BotConfig, BundledOrders, CliOptions, RoundReport, SgFilter, RpcRecord } from "./types"; +import { + SgFilter, + BotConfig, + RpcRecord, + CliOptions, + RoundReport, + BundledOrders, + OperationState, +} from "./types"; import { getChainConfig, getDataFetcher, @@ -231,13 +239,14 @@ export async function getConfig( export async function clear( config: BotConfig, bundledOrders: BundledOrders[][], + state: OperationState, tracer: Tracer, ctx: Context, ): Promise { const version = versions.node; const majorVersion = Number(version.slice(0, version.indexOf("."))); - if (majorVersion >= 18) return await processOrders(config, bundledOrders, tracer, ctx); + if (majorVersion >= 18) return await processOrders(config, bundledOrders, state, tracer, ctx); else throw `NodeJS v18 or higher is required for running the app, current version: ${version}`; } diff --git a/src/modes/index.ts b/src/modes/index.ts index 2014d16f..83acac27 100644 --- a/src/modes/index.ts +++ b/src/modes/index.ts @@ -1,10 +1,9 @@ +import { Contract } from "ethers"; import { PublicClient } from "viem"; import { DataFetcher } from "sushi"; import { Token } from "sushi/currency"; -import { BigNumber, Contract } from "ethers"; import { findOpp as findInterObOpp } from "./interOrderbook"; import { findOpp as findIntraObOpp } from "./intraOrderbook"; -import { publicActionsL2, walletActionsL2 } from "viem/op-stack"; import { findOppWithRetries as findRpOpp } from "./routeProcessor"; import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; @@ -29,6 +28,7 @@ export async function findOpp({ inputToEthPrice, outputToEthPrice, orderbooksOrders, + l1GasPrice, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -43,27 +43,8 @@ export async function findOpp({ outputToEthPrice: string; toToken: Token; fromToken: Token; + l1GasPrice: bigint; }): Promise { - try { - const gp = BigNumber.from(await viemClient.getGasPrice()) - .mul(config.gasPriceMultiplier) - .div("100") - .toBigInt(); - if (gp > gasPrice) gasPrice = gp; - } catch { - /**/ - } - - // if chain is special L2, get L1 gas price just before dryruns - let l1Signer; - let l1GasPrice = 0n; - if (config.isSpecialL2) { - try { - l1Signer = signer.extend(walletActionsL2()).extend(publicActionsL2()); - l1GasPrice = await l1Signer.getL1BaseFee(); - } catch {} - } - const promises = [ findRpOpp({ orderPairObject, @@ -77,7 +58,6 @@ export async function findOpp({ config, viemClient, l1GasPrice, - l1Signer, }), ...(!config.rpOnly ? [ @@ -90,6 +70,7 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1GasPrice, }), findInterObOpp({ orderPairObject, @@ -101,6 +82,7 @@ export async function findOpp({ config, viemClient, orderbooksOrders, + l1GasPrice, }), ] : []), diff --git a/src/modes/interOrderbook.ts b/src/modes/interOrderbook.ts index 31f67835..16c1002e 100644 --- a/src/modes/interOrderbook.ts +++ b/src/modes/interOrderbook.ts @@ -21,7 +21,6 @@ export async function dryrun({ outputToEthPrice, config, viemClient, - l1Signer, l1GasPrice, }: { config: BotConfig; @@ -34,7 +33,6 @@ export async function dryrun({ outputToEthPrice: string; opposingOrders: BundledOrders; maximumInput: BigNumber; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; @@ -116,7 +114,7 @@ export async function dryrun({ try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); l1Cost = estimation.l1Cost; gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); } catch (e) { @@ -161,7 +159,7 @@ export async function dryrun({ try { spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); @@ -235,7 +233,6 @@ export async function findOpp({ config, viemClient, orderbooksOrders, - l1Signer, l1GasPrice, }: { config: BotConfig; @@ -247,7 +244,6 @@ export async function findOpp({ gasPrice: bigint; inputToEthPrice: string; outputToEthPrice: string; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { if (!arb) throw undefined; @@ -307,7 +303,6 @@ export async function findOpp({ outputToEthPrice, config, viemClient, - l1Signer, l1GasPrice, }); }), @@ -378,7 +373,6 @@ export async function binarySearch({ outputToEthPrice, config, viemClient, - l1Signer, l1GasPrice, }: { config: BotConfig; @@ -391,7 +385,6 @@ export async function binarySearch({ outputToEthPrice: string; opposingOrders: BundledOrders; maximumInput: ethers.BigNumber; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { const spanAttributes = {}; @@ -416,7 +409,6 @@ export async function binarySearch({ outputToEthPrice, config, viemClient, - l1Signer, l1GasPrice, }), ); diff --git a/src/modes/intraOrderbook.ts b/src/modes/intraOrderbook.ts index d60b2171..995ddebc 100644 --- a/src/modes/intraOrderbook.ts +++ b/src/modes/intraOrderbook.ts @@ -28,7 +28,6 @@ export async function dryrun({ viemClient, inputBalance, outputBalance, - l1Signer, l1GasPrice, }: { config: BotConfig; @@ -41,7 +40,6 @@ export async function dryrun({ inputBalance: BigNumber; outputBalance: BigNumber; opposingOrder: TakeOrderDetails; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; @@ -112,7 +110,7 @@ export async function dryrun({ try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); l1Cost = estimation.l1Cost; gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); } catch (e) { @@ -167,7 +165,7 @@ export async function dryrun({ try { spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); @@ -246,7 +244,6 @@ export async function findOpp({ config, viemClient, orderbooksOrders, - l1Signer, l1GasPrice, }: { config: BotConfig; @@ -257,7 +254,6 @@ export async function findOpp({ gasPrice: bigint; inputToEthPrice: string; outputToEthPrice: string; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; @@ -326,7 +322,6 @@ export async function findOpp({ viemClient, inputBalance, outputBalance, - l1Signer, l1GasPrice, }); } catch (e: any) { diff --git a/src/modes/routeProcessor.ts b/src/modes/routeProcessor.ts index 2ee8ba69..b5deb3d3 100644 --- a/src/modes/routeProcessor.ts +++ b/src/modes/routeProcessor.ts @@ -41,7 +41,6 @@ export async function dryrun({ config, viemClient, hasPriceMatch, - l1Signer, l1GasPrice, }: { mode: number; @@ -57,7 +56,6 @@ export async function dryrun({ fromToken: Token; maximumInput: BigNumber; hasPriceMatch?: { value: boolean }; - l1Signer?: any; l1GasPrice?: bigint; }) { const spanAttributes: SpanAttrs = {}; @@ -178,7 +176,7 @@ export async function dryrun({ try { blockNumber = Number(await viemClient.getBlockNumber()); spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice, l1Signer); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); l1Cost = estimation.l1Cost; gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) @@ -227,13 +225,7 @@ export async function dryrun({ try { spanAttributes["blockNumber"] = blockNumber; - const estimation = await estimateGasCost( - rawtx, - signer, - config, - l1GasPrice, - l1Signer, - ); + const estimation = await estimateGasCost(rawtx, signer, config, l1GasPrice); gasLimit = ethers.BigNumber.from(estimation.gas) .mul(config.gasLimitMultiplier) .div(100); @@ -317,7 +309,6 @@ export async function findOpp({ config, viemClient, l1GasPrice, - l1Signer, }: { mode: number; config: BotConfig; @@ -331,7 +322,6 @@ export async function findOpp({ toToken: Token; fromToken: Token; l1GasPrice?: bigint; - l1Signer?: any; }): Promise { const spanAttributes: SpanAttrs = {}; const result: DryrunResult = { @@ -368,7 +358,6 @@ export async function findOpp({ viemClient, hasPriceMatch, l1GasPrice, - l1Signer, }); } catch (e: any) { // the fail reason can only be no route in case all hops fail reasons are no route @@ -402,7 +391,6 @@ export async function findOpp({ config, viemClient, l1GasPrice, - l1Signer, }); } catch (e: any) { // the fail reason can only be no route in case all hops fail reasons are no route @@ -446,7 +434,6 @@ export async function findOppWithRetries({ config, viemClient, l1GasPrice, - l1Signer, }: { config: BotConfig; orderPairObject: BundledOrders; @@ -458,7 +445,6 @@ export async function findOppWithRetries({ ethPrice: string; toToken: Token; fromToken: Token; - l1Signer?: any; l1GasPrice?: bigint; }): Promise { const spanAttributes: SpanAttrs = {}; @@ -484,7 +470,6 @@ export async function findOppWithRetries({ config, viemClient, l1GasPrice, - l1Signer, }), ); } diff --git a/src/processOrders.ts b/src/processOrders.ts index 05bcacd2..6915fe75 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -19,6 +19,7 @@ import { RoundReport, BundledOrders, BotDataFetcher, + OperationState, ProcessPairResult, } from "./types"; import { @@ -37,13 +38,12 @@ import { */ export enum ProcessPairHaltReason { FailedToQuote = 1, - FailedToGetGasPrice = 2, - FailedToGetEthPrice = 3, - FailedToGetPools = 4, - TxFailed = 5, - TxMineFailed = 6, - TxReverted = 7, - UnexpectedError = 8, + FailedToGetEthPrice = 2, + FailedToGetPools = 3, + TxFailed = 4, + TxMineFailed = 5, + TxReverted = 6, + UnexpectedError = 7, } /** @@ -65,6 +65,7 @@ export enum ProcessPairReportStatus { export const processOrders = async ( config: BotConfig, bundledOrders: BundledOrders[][], + state: OperationState, tracer: Tracer, ctx: Context, ): Promise => { @@ -85,7 +86,7 @@ export const processOrders = async ( try { const ownedOrders = await checkOwnedOrders(config, bundledOrders); if (ownedOrders.length) { - const failedFundings = await fundOwnedOrders(ownedOrders, config); + const failedFundings = await fundOwnedOrders(ownedOrders, config, state); const emptyOrders = ownedOrders.filter((v) => v.vaultBalance.isZero()); if (failedFundings.length || emptyOrders.length) { const message: string[] = []; @@ -229,6 +230,7 @@ export const processOrders = async ( orderbook, pair, orderbooksOrders: bundledOrders, + state, }); results.push({ settle, pair, orderPairObject }); span.end(); @@ -293,14 +295,6 @@ export const processOrders = async ( message = errorSnapshot(message, e.error); } span.setStatus({ code: SpanStatusCode.OK, message }); - } else if (e.reason === ProcessPairHaltReason.FailedToGetGasPrice) { - let message = pair + ": failed to get gas price"; - if (e.error) { - message = errorSnapshot(message, e.error); - span.recordException(e.error); - } - span.setAttribute("severity", ErrorSeverity.LOW); - span.setStatus({ code: SpanStatusCode.ERROR, message }); } else if (e.reason === ProcessPairHaltReason.FailedToGetPools) { let message = pair + ": failed to get pool details"; if (e.error) { @@ -430,6 +424,7 @@ export async function processPair(args: { orderbook: Contract; pair: string; orderbooksOrders: BundledOrders[][]; + state: OperationState; }): Promise<() => Promise> { const { config, @@ -443,6 +438,7 @@ export async function processPair(args: { orderbook, pair, orderbooksOrders, + state, } = args; const spanAttributes: SpanAttrs = {}; @@ -458,6 +454,7 @@ export async function processPair(args: { sellToken: orderPairObject.sellToken, }, }; + const gasPrice = ethers.BigNumber.from(state.gasPrice); spanAttributes["details.orders"] = orderPairObject.takeOrders.map((v) => v.id); spanAttributes["details.pair"] = pair; @@ -504,20 +501,6 @@ export async function processPair(args: { ratio: ethers.utils.formatUnits(orderPairObject.takeOrders[0].quote!.ratio), }); - // get gas price - let gasPrice; - try { - const gasPriceBigInt = await viemClient.getGasPrice(); - gasPrice = ethers.BigNumber.from(gasPriceBigInt).mul(config.gasPriceMultiplier).div("100"); - spanAttributes["details.gasPrice"] = gasPrice.toString(); - } catch (e) { - result.reason = ProcessPairHaltReason.FailedToGetGasPrice; - result.error = e; - return async () => { - throw result; - }; - } - // get pool details if ( !dataFetcher.fetchedPairPools.includes(pair) || @@ -617,6 +600,12 @@ export async function processPair(args: { } } + // record gas price for otel + spanAttributes["details.gasPrice"] = state.gasPrice.toString(); + if (state.l1GasPrice) { + spanAttributes["details.gasPriceL1"] = state.l1GasPrice.toString(); + } + // execute process to find opp through different modes let rawtx, oppBlockNumber, estimatedProfit; try { @@ -628,12 +617,13 @@ export async function processPair(args: { fromToken, toToken, signer, - gasPrice: gasPrice.toBigInt(), + gasPrice: state.gasPrice, config, viemClient, inputToEthPrice, outputToEthPrice, orderbooksOrders, + l1GasPrice: state.l1GasPrice, }); ({ rawtx, oppBlockNumber, estimatedProfit } = findOppResult.value!); diff --git a/src/tx.ts b/src/tx.ts index cc662adf..b9216539 100644 --- a/src/tx.ts +++ b/src/tx.ts @@ -1,6 +1,6 @@ -import { getL1Fee } from "./gas"; import { Token } from "sushi/currency"; import { Contract, ethers } from "ethers"; +import { getL1Fee, getTxFee } from "./gas"; import { addWatchedToken } from "./account"; import { containsNodeError, handleRevert } from "./error"; import { ProcessPairHaltReason, ProcessPairReportStatus } from "./processOrders"; @@ -167,9 +167,7 @@ export async function handleReceipt( time: number, ): Promise { const l1Fee = getL1Fee(receipt, config); - const actualGasCost = ethers.BigNumber.from(receipt.effectiveGasPrice) - .mul(receipt.gasUsed) - .add(l1Fee); + const actualGasCost = ethers.BigNumber.from(getTxFee(receipt, config)); const signerBalance = signer.BALANCE; signer.BALANCE = signer.BALANCE.sub(actualGasCost); diff --git a/src/types.ts b/src/types.ts index 2bab93c9..e712c69a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -198,6 +198,11 @@ export type BotConfig = { onFetchResponse?: (request: Response) => void; }; +export type OperationState = { + gasPrice: bigint; + l1GasPrice: bigint; +}; + export type Report = { status: ProcessPairReportStatus; tokenPair: string; diff --git a/test/account.test.js b/test/account.test.js index f8b2113e..549b9ad6 100644 --- a/test/account.test.js +++ b/test/account.test.js @@ -276,6 +276,7 @@ describe("Test accounts", async function () { ), ], ]); + const state = { gasPrice: 5n }; const config = { chain: { id: chainId }, mainAccount: { @@ -310,7 +311,7 @@ describe("Test accounts", async function () { }, }; - await sweepToEth(config); + await sweepToEth(config, state); assert.deepEqual(config.mainAccount.BOUNTY, []); }); @@ -360,6 +361,7 @@ describe("Test accounts", async function () { ), ], ]); + const state = { gasPrice: 5n }; const config = { chain: { id: chainId }, mainAccount: { @@ -414,7 +416,7 @@ describe("Test accounts", async function () { }, ]; - const result = await fundOwnedOrders(ownedOrders, config); + const result = await fundOwnedOrders(ownedOrders, config, state); assert.deepEqual(result, []); assert.ok( // (balance - gasCost - gasCost - sent topup) >= current balance (a bit lower than right side because of pool fee) diff --git a/test/cli.test.js b/test/cli.test.js index 5055cff8..d9b97771 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -202,8 +202,8 @@ describe("Test cli", async function () { route: "single", rpcRecords: { "https://rpc.ankr.com/polygon/": { - req: 1, - success: 1, + req: 2, + success: 2, failure: 0, cache: {}, }, diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index 8b653022..7adecf1e 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -262,7 +262,10 @@ for (let i = 0; i < testData.length; i++) { await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); - const { reports } = await clear(config, orders, tracer, ctx); + const state = { + gasPrice: await bot.getGasPrice(), + }; + const { reports } = await clear(config, orders, state, tracer, ctx); // should have cleared correct number of orders assert.ok(reports.length == tokens.length - 1, "Failed to clear all given orders"); @@ -600,7 +603,10 @@ for (let i = 0; i < testData.length; i++) { await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); - const { reports } = await clear(config, orders, tracer, ctx); + const state = { + gasPrice: await bot.getGasPrice(), + }; + const { reports } = await clear(config, orders, state, tracer, ctx); // should have cleared correct number of orders assert.ok( @@ -956,7 +962,10 @@ for (let i = 0; i < testData.length; i++) { await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); - const { reports } = await clear(config, orders, tracer, ctx); + const state = { + gasPrice: await bot.getGasPrice(), + }; + const { reports } = await clear(config, orders, state, tracer, ctx); // should have cleared correct number of orders assert.ok( diff --git a/test/gas.test.ts b/test/gas.test.ts index c5dbf777..548f5f27 100644 --- a/test/gas.test.ts +++ b/test/gas.test.ts @@ -1,8 +1,8 @@ import { assert } from "chai"; -import { estimateGasCost, getL1Fee } from "../src/gas"; +import { ChainId } from "sushi"; import { BotConfig, ViemClient } from "../src/types"; +import { estimateGasCost, getL1Fee, getTxFee } from "../src/gas"; import { createViemClient, getChainConfig } from "../src/config"; -import { ChainId } from "sushi"; describe("Test gas", async function () { it("should estimate gas correctly for L1 and L2 chains", async function () { @@ -22,7 +22,7 @@ describe("Test gas", async function () { }; // estimate gas as special L2 chain - const botconfig = { isSpecialL2: true } as any as BotConfig; + const botconfig = { isSpecialL2: true, gasPriceMultiplier: 100n } as any as BotConfig; const result1 = await estimateGasCost(tx, signer, botconfig, undefined, l1Signer); const expected1 = { gas: 55n, @@ -58,4 +58,22 @@ describe("Test gas", async function () { const expected = 43615200401n; // known L1 cost taken from the actual tx assert.equal(result, expected); }); + + it("should get tx fee", async function () { + const config = {} as any; + const receipt = { + effectiveGasPrice: 10n, + gasUsed: 5n, + } as any; + + // normal + let result = getTxFee(receipt, config); + assert.equal(result, 50n); + + // l2 chain + config.isSpecialL2 = true; + receipt.l1Fee = 50n; + result = getTxFee(receipt, config); + assert.equal(result, 100n); + }); }); diff --git a/test/processPair.test.js b/test/processPair.test.js index 6d06f6f7..e9109f17 100644 --- a/test/processPair.test.js +++ b/test/processPair.test.js @@ -43,6 +43,9 @@ describe("Test process pair", async function () { const config = JSON.parse(JSON.stringify(fixtureConfig)); config.rpc = ["http://localhost:8082/rpc"]; const quoteResponse = encodeQuoteResponse([[true, vaultBalance, ethers.constants.Zero]]); + const state = { + gasPrice: gasPrice.mul(107).div(100).toBigInt(), + }; beforeEach(() => { mockServer.start(8082); @@ -115,6 +118,7 @@ describe("Test process pair", async function () { accounts: [signer], fetchedPairPools: [], orderbooksOrders, + state, }) )(); const expected = { @@ -198,6 +202,7 @@ describe("Test process pair", async function () { accounts: [signer], fetchedPairPools: [], orderbooksOrders, + state, }) )(); const expected = { @@ -276,6 +281,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); const expected = { @@ -313,6 +319,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); @@ -336,54 +343,6 @@ describe("Test process pair", async function () { } }); - it("should fail to get gas price", async function () { - await mockServer.forPost("/rpc").thenSendJsonRpcResult(quoteResponse); - const evmError = { code: ethers.errors.CALL_EXCEPTION }; - viemClient.getGasPrice = async () => { - return Promise.reject(evmError); - }; - try { - await ( - await processPair({ - config, - orderPairObject, - viemClient, - dataFetcher, - signer, - flashbotSigner: undefined, - arb, - orderbook, - pair, - mainAccount: signer, - accounts: [signer], - fetchedPairPools: [], - }) - )(); - assert.fail("expected to reject, but resolved"); - } catch (error) { - const expected = { - report: { - status: ProcessPairReportStatus.NoOpportunity, - tokenPair: pair, - buyToken: orderPairObject.buyToken, - sellToken: orderPairObject.sellToken, - }, - gasCost: undefined, - reason: ProcessPairHaltReason.FailedToGetGasPrice, - error: evmError, - spanAttributes: { - "details.pair": pair, - "details.orders": [orderPairObject.takeOrders[0].id], - "details.quote": JSON.stringify({ - maxOutput: formatUnits(vaultBalance), - ratio: formatUnits(ethers.constants.Zero), - }), - }, - }; - assert.deepEqual(error, expected); - } - }); - it("should fail to get eth price", async function () { await mockServer.forPost("/rpc").thenSendJsonRpcResult(quoteResponse); config.gasCoveragePercentage = "100"; @@ -405,6 +364,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); @@ -422,7 +382,6 @@ describe("Test process pair", async function () { spanAttributes: { "details.pair": pair, "details.orders": [orderPairObject.takeOrders[0].id], - "details.gasPrice": gasPrice.mul(107).div(100).toString(), "details.marketQuote.num": 0.99699, "details.marketQuote.str": "0.99699", "details.quote": JSON.stringify({ @@ -456,6 +415,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); @@ -473,7 +433,6 @@ describe("Test process pair", async function () { spanAttributes: { "details.pair": pair, "details.orders": [orderPairObject.takeOrders[0].id], - "details.gasPrice": gasPrice.mul(107).div(100).toString(), "details.quote": JSON.stringify({ maxOutput: formatUnits(vaultBalance), ratio: formatUnits(ethers.constants.Zero), @@ -508,6 +467,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); @@ -618,6 +578,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); @@ -702,6 +663,7 @@ describe("Test process pair", async function () { mainAccount: signer, accounts: [signer], fetchedPairPools: [], + state, }) )(); assert.fail("expected to reject, but resolved"); From fba42bc7534b8c340340d39cc32288a286f8fcf0 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 27 Dec 2024 18:41:07 +0000 Subject: [PATCH 15/17] update tests --- test/gas.test.ts | 92 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 21 deletions(-) diff --git a/test/gas.test.ts b/test/gas.test.ts index 548f5f27..6e5d5c66 100644 --- a/test/gas.test.ts +++ b/test/gas.test.ts @@ -1,29 +1,32 @@ import { assert } from "chai"; -import { ChainId } from "sushi"; -import { BotConfig, ViemClient } from "../src/types"; -import { estimateGasCost, getL1Fee, getTxFee } from "../src/gas"; -import { createViemClient, getChainConfig } from "../src/config"; +import { OperationState, ViemClient } from "../src/types"; +import { estimateGasCost, getGasPrice, getL1Fee, getTxFee } from "../src/gas"; describe("Test gas", async function () { it("should estimate gas correctly for L1 and L2 chains", async function () { - // mock l1 signer + // mock config + const config = { + isSpecialL2: true, + gasPriceMultiplier: 100n, + } as any; + // mock L1 signer for L2 client const l1Signer = { getL1BaseFee: async () => 20n, estimateL1Gas: async () => 5n, }; - // mock normal signer + // mock normal L1 signer const signer = { estimateGas: async () => 55n, getGasPrice: async () => 2n, } as any as ViemClient; + // mock tx const tx = { data: "0x1234" as `0x${string}`, to: ("0x" + "1".repeat(40)) as `0x${string}`, }; // estimate gas as special L2 chain - const botconfig = { isSpecialL2: true, gasPriceMultiplier: 100n } as any as BotConfig; - const result1 = await estimateGasCost(tx, signer, botconfig, undefined, l1Signer); + const result1 = await estimateGasCost(tx, signer, config, undefined, l1Signer); const expected1 = { gas: 55n, gasPrice: 2n, @@ -34,9 +37,9 @@ describe("Test gas", async function () { }; assert.deepEqual(result1, expected1); - // estimate as usual chain - botconfig.isSpecialL2 = false; - const result2 = await estimateGasCost(tx, signer, botconfig); + // estimate as L1 chain + config.isSpecialL2 = false; + const result2 = await estimateGasCost(tx, signer, config); const expected2 = { gas: 55n, gasPrice: 2n, @@ -48,32 +51,79 @@ describe("Test gas", async function () { assert.deepEqual(result2, expected2); }); - it("should get tx L1 gas cost from receipt", async function () { - const config = getChainConfig(ChainId.BASE); - const viemclient = await createViemClient(ChainId.BASE, ["https://rpc.ankr.com/base"]); - const hash = "0x18219497dc46babfbdc58fad112bf01ed584148bf06727cc97cb105915fd96b0"; - const receipt = await viemclient.getTransactionReceipt({ hash }); + it("should get L1 fee from receipt", async function () { + // mock config + const config = { + isSpecialL2: true, + } as any; + + // chain is L1, no L1 fee in receipt + const receipt1 = { + effectiveGasPrice: 10n, + gasUsed: 5n, + } as any; + assert.equal(getL1Fee(receipt1, config), 0n); - const result = getL1Fee(receipt, config as any as BotConfig); - const expected = 43615200401n; // known L1 cost taken from the actual tx - assert.equal(result, expected); + // L2 + config.isSpecialL2 = true; + const receipt2 = { + effectiveGasPrice: 10n, + gasUsed: 5n, + l1Fee: 6n, + l1GasPrice: 2n, + l1GasUsed: 3n, + } as any; + assert.equal(getL1Fee(receipt2, config), 6n); }); it("should get tx fee", async function () { + // mock config and receipt const config = {} as any; const receipt = { effectiveGasPrice: 10n, gasUsed: 5n, } as any; - // normal + // L1 let result = getTxFee(receipt, config); assert.equal(result, 50n); - // l2 chain + // L2 config.isSpecialL2 = true; receipt.l1Fee = 50n; result = getTxFee(receipt, config); assert.equal(result, 100n); }); + + it("should get gas price", async function () { + const gasPrice = 10n; + const l1GasPrice = 2n; + // mock config and viem client + const config = { + isSpecialL2: false, + gasPriceMultiplier: 100n, + viemClient: { getGasPrice: async () => gasPrice }, + } as any; + + // test L1 chain + const state1: OperationState = { + gasPrice: 0n, + l1GasPrice: 0n, + }; + await getGasPrice(config, state1); + assert.equal(state1.gasPrice, gasPrice); + assert.equal(state1.l1GasPrice, 0n); + + // test L2 chain + config.isSpecialL2 = true; + config.viemClient.extend = () => config.viemClient; + config.viemClient.getL1BaseFee = async () => l1GasPrice; + const state2: OperationState = { + gasPrice: 0n, + l1GasPrice: 0n, + }; + await getGasPrice(config, state2); + assert.equal(state2.gasPrice, gasPrice); + assert.equal(state2.l1GasPrice, l1GasPrice); + }); }); From c7081604c280a16d8f9b98d5dd3c10575bab295e Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 2 Jan 2025 22:41:02 +0000 Subject: [PATCH 16/17] Update e2e.test.js --- test/e2e/e2e.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index 8b653022..5b5f94e7 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -73,7 +73,7 @@ for (let i = 0; i < testData.length; i++) { const tracer = provider.getTracer("arb-bot-tracer"); config.rpc = [rpc]; - const dataFetcherPromise = getDataFetcher(config, liquidityProviders, false); + const dataFetcherPromise = getDataFetcher(config, liquidityProviders, true); // run tests on each rp version for (let j = 0; j < rpVersions.length; j++) { From e6d7ce84299357f6413e4972a23c5b7c816102b6 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 3 Jan 2025 02:27:07 +0000 Subject: [PATCH 17/17] fix --- src/modes/routeProcessor.ts | 1 - test/findOpp.test.js | 33 ++++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/modes/routeProcessor.ts b/src/modes/routeProcessor.ts index db342e42..d0aa2a18 100644 --- a/src/modes/routeProcessor.ts +++ b/src/modes/routeProcessor.ts @@ -397,7 +397,6 @@ export async function findOpp({ } catch (e: any) { // the fail reason can only be no route in case all hops fail reasons are no route if (e.reason !== RouteProcessorDryrunHaltReason.NoRoute) noRoute = false; - delete e.spanAttributes["rawtx"]; allNoneNodeErrors.push(e?.value?.noneNodeError); extendSpanAttributes(spanAttributes, e.spanAttributes, "partial"); } diff --git a/test/findOpp.test.js b/test/findOpp.test.js index b87e00e1..a6d8fd7b 100644 --- a/test/findOpp.test.js +++ b/test/findOpp.test.js @@ -3,7 +3,7 @@ const testData = require("./data"); const { findOpp } = require("../src/modes"); const { orderbookAbi } = require("../src/abis"); const { errorSnapshot } = require("../src/error"); -const { clone, estimateProfit } = require("../src/utils"); +const { clone, estimateProfit, withBigintSerializer } = require("../src/utils"); const { ethers, utils: { formatUnits }, @@ -323,7 +323,7 @@ describe("Test find opp", async function () { assert.deepEqual(result, expected); }); - it("should NOT find opp", async function () { + it.only("should NOT find opp", async function () { const err = ethers.errors.UNPREDICTABLE_GAS_LIMIT; signer.estimateGas = async () => { return Promise.reject(err); @@ -351,7 +351,7 @@ describe("Test find opp", async function () { } catch (error) { const expectedTakeOrdersConfigStruct = { minimumInput: ethers.constants.One, - maximumInput: vaultBalance, + maximumInput: ethers.constants.MaxUint256, maximumIORatio: ethers.constants.MaxUint256, orders: [orderPairObject.takeOrders[0].takeOrder], data: expectedRouteData, @@ -370,16 +370,19 @@ describe("Test find opp", async function () { }, signedContext: [], }; - const rawtx = JSON.stringify({ - data: arb.interface.encodeFunctionData("arb3", [ - orderPairObject.orderbook, - expectedTakeOrdersConfigStruct, - task, - ]), - to: arb.address, - gasPrice, - from: signer.account.address, - }); + const rawtx = JSON.stringify( + { + data: arb.interface.encodeFunctionData("arb3", [ + orderPairObject.orderbook, + expectedTakeOrdersConfigStruct, + task, + ]), + to: arb.address, + gasPrice: gasPrice, + from: signer.account.address, + }, + withBigintSerializer, + ); const opposingMaxInput = vaultBalance .mul(orderPairObject.takeOrders[0].quote.ratio) .div(`1${"0".repeat(36 - orderPairObject.buyTokenDecimals)}`); @@ -398,7 +401,7 @@ describe("Test find opp", async function () { ]); const expectedTakeOrdersConfigStruct2 = { minimumInput: ethers.constants.One, - maximumInput: vaultBalance, + maximumInput: ethers.constants.MaxUint256, maximumIORatio: ethers.constants.MaxUint256, orders: [orderPairObject.takeOrders[0].takeOrder], data: ethers.utils.defaultAbiCoder.encode( @@ -456,7 +459,7 @@ describe("Test find opp", async function () { [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.blockNumber`]: oppBlockNumber, [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.rawtx`]: - JSON.stringify(rawtx2), + JSON.stringify(rawtx2, withBigintSerializer), [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.maxInput`]: vaultBalance.toString(), [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.error`]: