From ce82066e7cd695053c49aa03ed031614af440d5f Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 25 Oct 2024 02:08:22 +0000 Subject: [PATCH 01/32] init --- README.md | 4 + example.env | 2 + src/abis.ts | 1 + src/cli.ts | 128 +++++++++++----- src/index.ts | 27 ++-- src/processOrders.ts | 8 +- src/query.ts | 81 +++++++++- src/sg.ts | 157 ++++++++++++++++++- src/types.ts | 26 ++++ src/utils.ts | 122 ++++++++++++++- src/watcher.ts | 354 +++++++++++++++++++++++++++++++++++++++++++ test/e2e/e2e.test.js | 22 ++- test/sg.test.js | 42 +---- 13 files changed, 867 insertions(+), 107 deletions(-) create mode 100644 src/watcher.ts diff --git a/README.md b/README.md index 9c983b13..8e31fccd 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ Other optional arguments are: - `--self-fund-orders`, Specifies owned order to get funded once their vault goes below the specified threshold, example: token,vaultId,threshold,toptupamount;token,vaultId,threshold,toptupamount;... . Will override the 'SELF_FUND_ORDERS' in env variables - `-w` or `--wallet-count`, Number of wallet to submit transactions with, requirs `--mnemonic`. Will override the 'WALLET_COUNT' in env variables - `-t` or `--topup-amount`, The initial topup amount of excess wallets, requirs `--mnemonic`. Will override the 'TOPUP_AMOUNT' in env variables +- `--owner-profile`, Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables - `-V` or `--version`, output the version number - `-h` or `--help`, output usage information @@ -221,6 +222,9 @@ BOT_MIN_BALANCE= # Specifies owned order to get funded once their vault goes below the specified threshold # example: token,vaultId,threshold,toptupamount;token,vaultId,threshold,toptupamount;... SELF_FUND_ORDERS= + +# Specifies the owner limit, in form of owner1=limit,owner2=limit,... , example: 0x123456=12,0x3456=44 +OWNER_PROFILE= ``` If both env variables and CLI argument are set, the CLI arguments will be prioritized and override the env variables. diff --git a/example.env b/example.env index 2d78a29a..d2b41cc1 100644 --- a/example.env +++ b/example.env @@ -79,6 +79,8 @@ BOT_MIN_BALANCE= # example: token,vaultId,threshold,toptupamount;token,vaultId,threshold,toptupamount;... SELF_FUND_ORDERS= +# Specifies the owner limit, in form of owner1=limit,owner2=limit,... , example: 0x123456=12,0x3456=44 +OWNER_PROFILE= # test rpcs vars TEST_POLYGON_RPC= diff --git a/src/abis.ts b/src/abis.ts index 351262f2..5a12efc0 100644 --- a/src/abis.ts +++ b/src/abis.ts @@ -34,6 +34,7 @@ export const ClearConfig = export const orderbookAbi = [ `event AddOrderV2(address sender, bytes32 orderHash, ${OrderV3} order)`, `event AfterClear(address sender, ${ClearStateChange} clearStateChange)`, + `event RemoveOrderV2(address sender, bytes32 orderHash, ${OrderV3} order)`, "function vaultBalance(address owner, address token, uint256 vaultId) external view returns (uint256 balance)", `function deposit2(address token, uint256 vaultId, uint256 amount, ${TaskV1}[] calldata tasks) external`, `function addOrder2(${OrderConfigV3} calldata config, ${TaskV1}[] calldata tasks) external returns (bool stateChanged)`, diff --git a/src/cli.ts b/src/cli.ts index 9a979c01..531beb5c 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,13 +3,13 @@ import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; -import { sleep, getOrdersTokens } from "./utils"; +import { sleep, getOrdersTokens, prepareRoundProcessingOrders } from "./utils"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; -import { BotConfig, CliOptions, ViemClient } from "./types"; +import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; @@ -28,6 +28,9 @@ import { ConsoleSpanExporter, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; +import { handleNewLogs, watchAllOrderbooks, WatchedOrderbookOrders } from "./watcher"; +import { SgOrder } from "./query"; +import { getOrderbookOwnersProfileMapFromSg } from "./sg"; config(); @@ -56,6 +59,9 @@ const ENV_OPTIONS = { topupAmount: process?.env?.TOPUP_AMOUNT, botMinBalance: process?.env?.BOT_MIN_BALANCE, selfFundOrders: process?.env?.SELF_FUND_ORDERS, + ownerProfile: process?.env?.OWNER_PROFILE + ? Array.from(process?.env?.OWNER_PROFILE.matchAll(/[^,\s]+/g)).map((v) => v[0]) + : undefined, rpc: process?.env?.RPC_URL ? Array.from(process?.env?.RPC_URL.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, @@ -158,6 +164,10 @@ const getOptions = async (argv: any, version?: string) => { "--self-fund-orders ", "Specifies owned order to get funded once their vault goes below the specified threshold, example: token,vaultId,threshold,toptupamount;token,vaultId,threshold,toptupamount;... . Will override the 'SELF_FUND_ORDERS' in env variables", ) + .option( + "--owner-profile ", + "Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables", + ) .description( [ "A NodeJS app to find and take arbitrage trades for Rain Orderbook orders against some DeFi liquidity providers, requires NodeJS v18 or higher.", @@ -193,7 +203,19 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.topupAmount = cmdOptions.topupAmount || ENV_OPTIONS.topupAmount; cmdOptions.selfFundOrders = cmdOptions.selfFundOrders || ENV_OPTIONS.selfFundOrders; cmdOptions.botMinBalance = cmdOptions.botMinBalance || ENV_OPTIONS.botMinBalance; + cmdOptions.ownerProfile = cmdOptions.ownerProfile || ENV_OPTIONS.ownerProfile; cmdOptions.bundle = cmdOptions.bundle ? ENV_OPTIONS.bundle : false; + if (cmdOptions.ownerProfile) { + const profiles: Record = {}; + cmdOptions.ownerProfile.forEach((v: string) => { + const parsed = v.split("="); + if (parsed.length !== 2) throw "Invalid owner profile"; + if (!/^[0-9]+$/.test(parsed[1])) + throw "Invalid owner profile limit, must be an integer gte 0"; + profiles[parsed[0].toLowerCase()] = Number(parsed[1]); + }); + cmdOptions.ownerProfile = profiles; + } if (cmdOptions.lps) { cmdOptions.lps = Array.from((cmdOptions.lps as string).matchAll(/[^,\s]+/g)).map( (v) => v[0], @@ -220,43 +242,45 @@ export const arbRound = async ( roundCtx: Context, options: CliOptions, config: BotConfig, + bundledOrders: BundledOrders[][], ) => { return await tracer.startActiveSpan("process-orders", {}, roundCtx, async (span) => { const ctx = trace.setSpan(context.active(), span); - let ordersDetails; + // let ordersDetails; + options; try { - try { - ordersDetails = await getOrderDetails( - options.subgraph, - { - orderHash: options.orderHash, - orderOwner: options.orderOwner, - orderbook: options.orderbookAddress, - }, - span, - config.timeout, - ); - if (!ordersDetails.length) { - span.setStatus({ code: SpanStatusCode.OK, message: "found no orders" }); - span.end(); - return { txs: [], foundOpp: false, avgGasCost: undefined }; - } - } catch (e: any) { - const snapshot = errorSnapshot("", e); - span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); - span.recordException(e); - span.setAttribute("didClear", false); - span.setAttribute("foundOpp", false); - span.end(); - return { txs: [], foundOpp: false, avgGasCost: undefined }; - } + // try { + // ordersDetails = await getOrderDetails( + // options.subgraph, + // { + // orderHash: options.orderHash, + // orderOwner: options.orderOwner, + // orderbook: options.orderbookAddress, + // }, + // span, + // config.timeout, + // ); + // if (!ordersDetails.length) { + // span.setStatus({ code: SpanStatusCode.OK, message: "found no orders" }); + // span.end(); + // return { txs: [], foundOpp: false, avgGasCost: undefined }; + // } + // } catch (e: any) { + // const snapshot = errorSnapshot("", e); + // span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); + // span.recordException(e); + // span.setAttribute("didClear", false); + // span.setAttribute("foundOpp", false); + // span.end(); + // return { txs: [], foundOpp: false, avgGasCost: undefined }; + // } try { let txs; let foundOpp = false; const { reports = [], avgGasCost = undefined } = await clear( config, - ordersDetails, + bundledOrders, tracer, ctx, ); @@ -363,7 +387,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? throw "expected a valid value for --bot-min-balance, it should be an number greater than 0"; } const poolUpdateInterval = _poolUpdateInterval * 60 * 1000; - let ordersDetails: any[] = []; + let ordersDetails: SgOrder[] = []; if (!process?.env?.TEST) for (let i = 0; i < 20; i++) { try { @@ -378,7 +402,8 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? else throw e; } } - options.tokens = getOrdersTokens(ordersDetails); + const tokens = getOrdersTokens(ordersDetails); + options.tokens = tokens; // get config const config = await getConfig( @@ -395,6 +420,13 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? options: options as CliOptions, poolUpdateInterval, config, + orderbooksOwnersProfileMap: await getOrderbookOwnersProfileMapFromSg( + ordersDetails, + config.watchClient, + tokens, + (options as CliOptions).ownerProfile, + ), + tokens, }; } @@ -433,9 +465,8 @@ export const main = async (argv: any, version?: string) => { const tracer = provider.getTracer("arb-bot-tracer"); // parse cli args and startup bot configuration - const { roundGap, options, poolUpdateInterval, config } = await tracer.startActiveSpan( - "startup", - async (startupSpan) => { + const { roundGap, options, poolUpdateInterval, config, orderbooksOwnersProfileMap, tokens } = + await tracer.startActiveSpan("startup", async (startupSpan) => { const ctx = trace.setSpan(context.active(), startupSpan); try { const result = await startup(argv, version, tracer, ctx); @@ -459,8 +490,19 @@ export const main = async (argv: any, version?: string) => { // reject the promise that makes the cli process to exit with error return Promise.reject(e); } - }, + }); + + const obs: string[] = []; + const watchedOrderbooksOrders: Record = {}; + orderbooksOwnersProfileMap.forEach((_, ob) => { + obs.push(ob.toLowerCase()); + }); + const orderbooksUnwatchers = watchAllOrderbooks( + obs, + config.watchClient, + watchedOrderbooksOrders, ); + orderbooksUnwatchers; const day = 24 * 60 * 60 * 1000; let lastGasReset = Date.now() + day; @@ -534,8 +576,22 @@ export const main = async (argv: any, version?: string) => { update = true; } try { + const bundledOrders = prepareRoundProcessingOrders(orderbooksOwnersProfileMap); + await handleNewLogs( + orderbooksOwnersProfileMap, + watchedOrderbooksOrders, + config.viemClient as any as ViemClient, + tokens, + (options as CliOptions).ownerProfile, + ); await rotateProviders(config, update); - const roundResult = await arbRound(tracer, roundCtx, options, config); + const roundResult = await arbRound( + tracer, + roundCtx, + options, + config, + bundledOrders, + ); let txs, foundOpp, roundAvgGasCost; if (roundResult) { txs = roundResult.txs; diff --git a/src/index.ts b/src/index.ts index 91037e77..3c301a18 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,10 +7,10 @@ import { processLps } from "./utils"; import { initAccounts } from "./account"; import { processOrders } from "./processOrders"; import { Context, Span } from "@opentelemetry/api"; -import { getQuery, statusCheckQuery } from "./query"; +import { getQuery, SgOrder, statusCheckQuery } from "./query"; import { checkSgStatus, handleSgResults } from "./sg"; import { Tracer } from "@opentelemetry/sdk-trace-base"; -import { BotConfig, CliOptions, RoundReport, SgFilter } from "./types"; +import { BotConfig, BundledOrders, CliOptions, RoundReport, SgFilter } from "./types"; import { createViemClient, getChainConfig, getDataFetcher } from "./config"; /** @@ -26,9 +26,9 @@ export async function getOrderDetails( sgFilters?: SgFilter, span?: Span, timeout?: number, -): Promise { +): Promise { const hasjson = false; - const ordersDetails: any[] = []; + const ordersDetails: SgOrder[] = []; const isInvalidSg = !Array.isArray(sgs) || sgs.length === 0; if (isInvalidSg) throw "type of provided sources are invalid"; @@ -56,16 +56,11 @@ export async function getOrderDetails( availableSgs.forEach((v) => { if (v && typeof v === "string") promises.push( - axios.post( + getQuery( v, - { - query: getQuery( - sgFilters?.orderHash, - sgFilters?.orderOwner, - sgFilters?.orderbook, - ), - }, - { headers: { "Content-Type": "application/json" }, timeout }, + sgFilters?.orderHash, + sgFilters?.orderOwner, + sgFilters?.orderbook, ), ); }); @@ -146,6 +141,7 @@ export async function getConfig( const config = getChainConfig(chainId) as any as BotConfig; const lps = processLps(options.lps); const viemClient = await createViemClient(chainId, rpcUrls, false, undefined, options.timeout); + const watchClient = await createViemClient(chainId, rpcUrls, false, undefined, options.timeout); const dataFetcher = await getDataFetcher(viemClient as any as PublicClient, lps, false); if (!config) throw `Cannot find configuration for the network with chain id: ${chainId}`; @@ -166,6 +162,7 @@ export async function getConfig( config.dataFetcher = dataFetcher; config.watchedTokens = options.tokens ?? []; config.selfFundOrders = options.selfFundOrders; + config.watchClient = watchClient; // init accounts const { mainAccount, accounts } = await initAccounts(walletKey, config, options, tracer, ctx); @@ -186,14 +183,14 @@ export async function getConfig( */ export async function clear( config: BotConfig, - ordersDetails: any[], + bundledOrders: BundledOrders[][], tracer: Tracer, ctx: Context, ): Promise { const version = versions.node; const majorVersion = Number(version.slice(0, version.indexOf("."))); - if (majorVersion >= 18) return await processOrders(config, ordersDetails, tracer, ctx); + if (majorVersion >= 18) return await processOrders(config, bundledOrders, tracer, ctx); else throw `NodeJS v18 or higher is required for running the app, current version: ${version}`; } diff --git a/src/processOrders.ts b/src/processOrders.ts index 9a33238a..5b73c1bf 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -27,7 +27,7 @@ import { getEthPrice, quoteOrders, routeExists, - bundleOrders, + // bundleOrders, PoolBlackList, getMarketQuote, getTotalIncome, @@ -68,7 +68,7 @@ export enum ProcessPairReportStatus { */ export const processOrders = async ( config: BotConfig, - ordersDetails: any[], + bundledOrders: BundledOrders[][], tracer: Tracer, ctx: Context, ): Promise => { @@ -85,7 +85,7 @@ export const processOrders = async ( } // prepare orders - const bundledOrders = bundleOrders(ordersDetails, false, true); + // const bundledOrders = bundleOrders(ordersDetails, false, true); // check owned vaults and top them up if necessary await tracer.startActiveSpan("handle-owned-vaults", {}, ctx, async (span) => { @@ -190,7 +190,7 @@ export const processOrders = async ( undefined, privateKeyToAccount( ethers.utils.hexlify( - ethers.utils.hexlify(signer.account.getHdKey().privateKey!), + signer.account.getHdKey().privateKey!, ) as `0x${string}`, ), config.timeout, diff --git a/src/query.ts b/src/query.ts index 8dfbf4d9..c2c18d03 100644 --- a/src/query.ts +++ b/src/query.ts @@ -1,3 +1,35 @@ +import axios from "axios"; + +export type SgOrder = { + id: string; + owner: string; + orderHash: string; + orderBytes: string; + active: boolean; + nonce: string; + orderbook: { + id: string; + }; + inputs: { + balance: string; + vaultId: string; + token: { + address: string; + decimals: string | number; + symbol: string; + }; + }[]; + outputs: { + balance: string; + vaultId: string; + token: { + address: string; + decimals: string | number; + symbol: string; + }; + }[]; +}; + /** * Method to get the subgraph query body with optional filters * @param orderHash - The order hash to apply as filter @@ -5,12 +37,17 @@ * @param orderbook - The orderbook address * @returns the query string */ -export function getQuery(orderHash?: string, owner?: string, orderbook?: string): string { +export function getQueryPaginated( + skip: number, + orderHash?: string, + owner?: string, + orderbook?: string, +): string { const ownerFilter = owner ? `, owner: "${owner.toLowerCase()}"` : ""; const orderHashFilter = orderHash ? `, orderHash: "${orderHash.toLowerCase()}"` : ""; const orderbookFilter = orderbook ? `, orderbook: "${orderbook.toLowerCase()}"` : ""; return `{ - orders(where: {active: true${orderbookFilter}${orderHashFilter}${ownerFilter}}) { + orders(first: 100, skip: ${skip}, where: {active: true${orderbookFilter}${orderHashFilter}${ownerFilter}}) { id owner orderHash @@ -42,6 +79,46 @@ export function getQuery(orderHash?: string, owner?: string, orderbook?: string) }`; } +/** + * Get all active orders from a subgraph, with optional filters + * @param subgraph - Subgraph url + * @param orderHash - orderHash filter + * @param owner - owner filter + * @param orderbook - orderbook filter + * @param timeout - timeout + */ +export async function getQuery( + subgraph: string, + orderHash?: string, + owner?: string, + orderbook?: string, + timeout?: number, +): Promise { + const result: any[] = []; + let skip = 0; + for (;;) { + const res = await axios.post( + subgraph, + { + query: getQueryPaginated(skip, orderHash, owner, orderbook), + }, + { headers: { "Content-Type": "application/json" }, timeout }, + ); + if (res?.data?.data?.orders?.length) { + const orders = res.data.data.orders; + result.push(...orders); + if (orders.length < 100) { + break; + } else { + skip += 100; + } + } else { + break; + } + } + return result; +} + export const orderbooksQuery = `{ orderbooks { id diff --git a/src/sg.ts b/src/sg.ts index 4fc4dd94..aa745ddb 100644 --- a/src/sg.ts +++ b/src/sg.ts @@ -1,7 +1,20 @@ import axios from "axios"; import { ErrorSeverity } from "./error"; import { Span } from "@opentelemetry/api"; -import { orderbooksQuery } from "./query"; +import { orderbooksQuery, SgOrder } from "./query"; +import { getTokenSymbol } from "./utils"; +import { + Order, + OrderbooksOwnersProfileMap, + OrdersProfileMap, + OwnersProfileMap, + Pair, + TokenDetails, + ViemClient, +} from "./types"; +import { OrderV3 } from "./abis"; +import { decodeAbiParameters, parseAbiParameters } from "viem"; +import { toOrder } from "./watcher"; /** * Checks a subgraph health status and records the result in an object or throws @@ -92,8 +105,8 @@ export function handleSgResults( const ordersDetails: any[] = []; for (let i = 0; i < responses.length; i++) { const res = responses[i]; - if (res.status === "fulfilled" && res?.value?.data?.data?.orders) { - ordersDetails.push(...res.value.data.data.orders); + if (res.status === "fulfilled" && res?.value) { + ordersDetails.push(...res.value); } else if (res.status === "rejected") { reasons[availableSgs[i]] = res.reason; } @@ -138,3 +151,141 @@ export async function getSgOrderbooks(url: string): Promise { throw msg.join("\n"); } } + +export async function getPairs( + orderStruct: Order, + viemClient: ViemClient, + tokens: TokenDetails[], + orderDetails?: SgOrder, +): Promise { + const pairs: Pair[] = []; + for (let j = 0; j < orderStruct.validOutputs.length; j++) { + const _output = orderStruct.validOutputs[j]; + let _outputSymbol = orderDetails?.outputs?.find( + (v) => v.token.address.toLowerCase() === _output.token.toLowerCase(), + )?.token?.symbol; + if (!_outputSymbol) { + const symbol = tokens.find( + (v) => v.address.toLowerCase() === _output.token.toLowerCase(), + )?.symbol; + if (!symbol) { + _outputSymbol = await getTokenSymbol(_output.token, viemClient); + } else { + _outputSymbol = symbol; + } + } else { + if (!tokens.find((v) => v.address.toLowerCase() === _output.token.toLowerCase())) { + tokens.push({ + address: _output.token.toLowerCase(), + symbol: _outputSymbol, + decimals: _output.decimals, + }); + } + } + + for (let k = 0; k < orderStruct.validInputs.length; k++) { + const _input = orderStruct.validInputs[k]; + let _inputSymbol = orderDetails?.inputs?.find( + (v) => v.token.address.toLowerCase() === _input.token.toLowerCase(), + )?.token?.symbol; + if (!_inputSymbol) { + const symbol = tokens.find( + (v) => v.address.toLowerCase() === _input.token.toLowerCase(), + )?.symbol; + if (!symbol) { + _inputSymbol = await getTokenSymbol(_input.token, viemClient); + } else { + _inputSymbol = symbol; + } + } else { + if (!tokens.find((v) => v.address.toLowerCase() === _input.token.toLowerCase())) { + tokens.push({ + address: _input.token.toLowerCase(), + symbol: _inputSymbol, + decimals: _input.decimals, + }); + } + } + + if (_input.token.toLowerCase() !== _output.token.toLowerCase()) + pairs.push({ + buyToken: _input.token.toLowerCase(), + buyTokenSymbol: _inputSymbol, + buyTokenDecimals: _input.decimals, + sellToken: _output.token.toLowerCase(), + sellTokenSymbol: _outputSymbol, + sellTokenDecimals: _output.decimals, + takeOrder: { + order: orderStruct, + inputIOIndex: k, + outputIOIndex: j, + signedContext: [], + }, + }); + } + } + return pairs; +} +/** + * Get a map of per owner orders per orderbook + * @param ordersDetails - Order details queried from subgraph + */ +export async function getOrderbookOwnersProfileMapFromSg( + ordersDetails: SgOrder[], + viemClient: ViemClient, + tokens: TokenDetails[], + ownerLimits?: Record, +): Promise { + const orderbookOwnersProfileMap: OrderbooksOwnersProfileMap = new Map(); + for (let i = 0; i < ordersDetails.length; i++) { + const orderDetails = ordersDetails[i]; + const orderbook = orderDetails.orderbook.id.toLowerCase(); + const orderStruct = toOrder( + decodeAbiParameters( + parseAbiParameters(OrderV3), + orderDetails.orderBytes as `0x${string}`, + )[0], + ); + const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + if (!ownerProfile.orders.has(orderDetails.orderHash.toLowerCase())) { + ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + const ownerProfileMap: OwnersProfileMap = new Map(); + ownerProfileMap.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); + } + } + return orderbookOwnersProfileMap; +} diff --git a/src/types.ts b/src/types.ts index 77bb8717..dcc459c9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -44,6 +44,7 @@ export type CliOptions = { bundle: boolean; selfFundOrders?: SelfFundOrder[]; tokens?: TokenDetails[]; + ownerProfile?: Record; }; export type TokenDetails = { @@ -65,6 +66,7 @@ export type BundledOrders = { export type TakeOrderDetails = { id: string; + // active: boolean; quote?: { maxOutput: BigNumber; ratio: BigNumber; @@ -99,6 +101,29 @@ export type Order = { validOutputs: IO[]; }; +export type Pair = { + buyToken: string; + buyTokenDecimals: number; + buyTokenSymbol: string; + sellToken: string; + sellTokenDecimals: number; + sellTokenSymbol: string; + takeOrder: TakeOrder; +}; +export type OrderProfile = { + active: boolean; + order: Order; + takeOrders: Pair[]; + consumedTakeOrders: Pair[]; +}; +export type OwnerProfile = { + limit: number; + orders: OrdersProfileMap; +}; +export type OrdersProfileMap = Map; +export type OwnersProfileMap = Map; +export type OrderbooksOwnersProfileMap = Map; + export type ViemClient = WalletClient & PublicActions & { BALANCE: BigNumber; BOUNTY: TokenDetails[] }; @@ -139,6 +164,7 @@ export type BotConfig = { mainAccount: ViemClient; accounts: ViemClient[]; selfFundOrders?: SelfFundOrder[]; + watchClient: ViemClient; }; export type Report = { diff --git a/src/utils.ts b/src/utils.ts index 45555319..3029de7c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -7,8 +7,18 @@ import { BigNumber, BigNumberish, ethers } from "ethers"; import { erc20Abi, orderbookAbi, OrderV3 } from "./abis"; import { parseAbi, PublicClient, TransactionReceipt } from "viem"; import { doQuoteTargets, QuoteTarget } from "@rainlanguage/orderbook/quote"; -import { BotConfig, BundledOrders, OwnedOrder, TakeOrder, TokenDetails } from "./types"; +import { + BotConfig, + BundledOrders, + OrderbooksOwnersProfileMap, + OwnedOrder, + Pair, + TakeOrder, + TokenDetails, + ViemClient, +} from "./types"; import { DataFetcher, DataFetcherOptions, LiquidityProviders, Router } from "sushi/router"; +import { SgOrder } from "./query"; export function RPoolFilter(pool: any) { return !BlackList.includes(pool.address) && !BlackList.includes(pool.address.toLowerCase()); @@ -633,6 +643,7 @@ export const bundleOrders = ( if (pair && _bundle) pair.takeOrders.push({ id: orderDetails.orderHash, + // active: true, takeOrder: { order: orderStruct, inputIOIndex: k, @@ -652,6 +663,7 @@ export const bundleOrders = ( takeOrders: [ { id: orderDetails.orderHash, + // active: true, takeOrder: { order: orderStruct, inputIOIndex: k, @@ -1078,7 +1090,7 @@ export const getRpSwap = async ( } }; -export function getOrdersTokens(ordersDetails: any[]): TokenDetails[] { +export function getOrdersTokens(ordersDetails: SgOrder[]): TokenDetails[] { const tokens: TokenDetails[] = []; for (let i = 0; i < ordersDetails.length; i++) { const orderDetails = ordersDetails[i]; @@ -1091,12 +1103,12 @@ export function getOrdersTokens(ordersDetails: any[]): TokenDetails[] { const _output = orderStruct.validOutputs[j]; const _outputSymbol = orderDetails.outputs.find( (v: any) => v.token.address.toLowerCase() === _output.token.toLowerCase(), - ).token.symbol; + )?.token?.symbol; if (!tokens.find((v) => v.address === _output.token.toLowerCase())) { tokens.push({ address: _output.token.toLowerCase(), decimals: _output.decimals, - symbol: _outputSymbol, + symbol: _outputSymbol ?? "UnknownSymbol", }); } } @@ -1104,12 +1116,12 @@ export function getOrdersTokens(ordersDetails: any[]): TokenDetails[] { const _input = orderStruct.validInputs[k]; const _inputSymbol = orderDetails.inputs.find( (v: any) => v.token.address.toLowerCase() === _input.token.toLowerCase(), - ).token.symbol; + )?.token?.symbol; if (!tokens.find((v) => v.address === _input.token.toLowerCase())) { tokens.push({ address: _input.token.toLowerCase(), decimals: _input.decimals, - symbol: _inputSymbol, + symbol: _inputSymbol ?? "UnknownSymbol", }); } } @@ -1277,3 +1289,101 @@ export function getMarketQuote( }; } } + +/** + * Get token symbol + * @param address - The address of token + * @param viemClient - The viem client + */ +export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { + try { + return await viemClient.readContract({ + address: address as `0x${string}`, + abi: parseAbi(erc20Abi), + functionName: "symbol", + }); + } catch (error) { + return "Unknownsymbol"; + } +} + +export function prepareRoundProcessingOrders( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, +): BundledOrders[][] { + const result: BundledOrders[][] = []; + for (const [orderbook, ownersProfileMap] of orderbooksOwnersProfileMap) { + const orderbookBundledOrders: BundledOrders[] = []; + for (const [, ownerProfile] of ownersProfileMap) { + let consumedLimit = ownerProfile.limit; + const activeOrdersProfiles = Array.from(ownerProfile.orders).filter((v) => v[1].active); + const remainingOrdersPairs = activeOrdersProfiles.filter( + (v) => v[1].takeOrders.length > 0, + ); + if (remainingOrdersPairs.length === 0) { + for (const [, orderProfile] of activeOrdersProfiles) { + orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); + } + for (const [orderHash, orderProfile] of activeOrdersProfiles) { + const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); + consumedLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + } + } else { + for (const [orderHash, orderProfile] of remainingOrdersPairs) { + const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); + consumedLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + } + } + } + result.push(orderbookBundledOrders); + } + return result; +} + +function gatherPairs( + orderbook: string, + orderHash: string, + pairs: Pair[], + bundledOrders: BundledOrders[], +) { + for (const pair of pairs) { + const bundleOrder = bundledOrders.find( + (v) => + v.buyToken.toLowerCase() === pair.buyToken.toLowerCase() && + v.buyTokenDecimals === pair.buyTokenDecimals && + v.buyTokenSymbol === pair.buyTokenSymbol && + v.sellToken.toLowerCase() === pair.sellToken.toLowerCase() && + v.sellTokenDecimals === pair.sellTokenDecimals && + v.sellTokenSymbol === pair.sellTokenSymbol, + ); + if (bundleOrder) { + if ( + !bundleOrder.takeOrders.find((v) => v.id.toLowerCase() === orderHash.toLowerCase()) + ) { + bundleOrder.takeOrders.push({ + id: orderHash, + takeOrder: pair.takeOrder, + }); + } + } else { + bundledOrders.push({ + orderbook, + buyToken: pair.buyToken, + buyTokenDecimals: pair.buyTokenDecimals, + buyTokenSymbol: pair.buyTokenSymbol, + sellToken: pair.sellToken, + sellTokenDecimals: pair.sellTokenDecimals, + sellTokenSymbol: pair.sellTokenSymbol, + takeOrders: [ + { + id: orderHash, + takeOrder: pair.takeOrder, + }, + ], + }); + } + } +} diff --git a/src/watcher.ts b/src/watcher.ts new file mode 100644 index 00000000..51bd333a --- /dev/null +++ b/src/watcher.ts @@ -0,0 +1,354 @@ +import { getPairs } from "./sg"; +import { orderbookAbi as abi } from "./abis"; +import { parseAbi, WatchContractEventReturnType } from "viem"; +import { + Order, + ViemClient, + TokenDetails, + OrdersProfileMap, + OwnersProfileMap, + OrderbooksOwnersProfileMap, +} from "./types"; +import { hexlify } from "ethers/lib/utils"; + +type OrderEventLog = { + sender: `0x${string}`; + orderHash: `0x${string}`; + order: { + owner: `0x${string}`; + evaluable: { + interpreter: `0x${string}`; + store: `0x${string}`; + bytecode: `0x${string}`; + }; + validInputs: readonly { + token: `0x${string}`; + decimals: number; + vaultId: bigint; + }[]; + validOutputs: readonly { + token: `0x${string}`; + decimals: number; + vaultId: bigint; + }[]; + nonce: `0x${string}`; + }; +}; +export type OrderArgsLog = { + sender: `0x${string}`; + orderHash: `0x${string}`; + order: Order; +}; +export type WatchedOrderbookOrders = { addOrders: OrderArgsLog[]; removeOrders: OrderArgsLog[] }; + +function toOrderArgsLog(orderLog: OrderEventLog): OrderArgsLog { + return { + sender: orderLog.sender, + orderHash: orderLog.orderHash, + order: toOrder(orderLog.order), + }; +} + +export function toOrder(orderLog: any): Order { + return { + owner: orderLog.owner, + nonce: orderLog.nonce, + evaluable: { + interpreter: orderLog.evaluable.interpreter, + store: orderLog.evaluable.store, + bytecode: orderLog.evaluable.bytecode, + }, + validInputs: orderLog.validInputs.map((v: any) => ({ + token: v.token.toLowerCase(), + decimals: v.decimals, + vaultId: hexlify(v.vaultId), + })), + validOutputs: orderLog.validOutputs.map((v: any) => ({ + token: v.token.toLowerCase(), + decimals: v.decimals, + vaultId: hexlify(v.vaultId), + })), + }; +} + +export const orderbookAbi = parseAbi(abi); +export type UnwatchOrderbook = { + unwatchAddOrder: WatchContractEventReturnType; + unwatchRemoveOrder: WatchContractEventReturnType; +}; + +export function watchOrderbook( + orderbook: string, + viemClient: ViemClient, + watchedOrderbookOrders: WatchedOrderbookOrders, +): UnwatchOrderbook { + const unwatchAddOrder = viemClient.watchContractEvent({ + address: orderbook as `0x${string}`, + abi: orderbookAbi, + eventName: "AddOrderV2", + onLogs: (logs) => { + logs.forEach((log) => { + if (log) { + watchedOrderbookOrders.addOrders.push( + toOrderArgsLog(log.args as any as OrderEventLog), + ); + } + }); + }, + }); + + const unwatchRemoveOrder = viemClient.watchContractEvent({ + address: orderbook as `0x${string}`, + abi: orderbookAbi, + eventName: "RemoveOrderV2", + onLogs: (logs) => { + logs.forEach((log) => { + if (log) { + watchedOrderbookOrders.removeOrders.push( + toOrderArgsLog(log.args as any as OrderEventLog), + ); + } + }); + }, + }); + + return { + unwatchAddOrder, + unwatchRemoveOrder, + }; +} + +export function watchAllOrderbooks( + orderbooks: string[], + viemClient: ViemClient, + watchedOrderbooksOrders: Record, +): Record { + const result: Record = {}; + for (const ob of orderbooks) { + if (!watchedOrderbooksOrders[ob]) + watchedOrderbooksOrders[ob] = { addOrders: [], removeOrders: [] }; + const res = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); + result[ob] = res; + } + return result; +} + +export function unwatchAll(watchers: Record) { + for (const ob in watchers) { + watchers[ob].unwatchAddOrder(); + watchers[ob].unwatchRemoveOrder(); + } +} + +export async function handleNewLogs( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + watchedOrderbooksOrders: Record, + viemClient: ViemClient, + tokens: TokenDetails[], + ownerLimits?: Record, +) { + for (const ob in watchedOrderbooksOrders) { + const watchedOrderbookLogs = watchedOrderbooksOrders[ob]; + await handleAddOrders( + ob, + watchedOrderbookLogs.addOrders.splice(0), + orderbooksOwnersProfileMap, + viemClient, + tokens, + ownerLimits, + ); + handleRemoveOrders( + ob, + watchedOrderbookLogs.removeOrders.splice(0), + orderbooksOwnersProfileMap, + ); + } +} + +/** + * Get a map of per owner orders per orderbook + * @param ordersDetails - Order details queried from subgraph + */ +export async function handleAddOrders( + orderbook: string, + addOrders: OrderArgsLog[], + orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, + viemClient: ViemClient, + tokens: TokenDetails[], + ownerLimits?: Record, +) { + orderbook = orderbook.toLowerCase(); + for (let i = 0; i < addOrders.length; i++) { + const addOrderLog = addOrders[i]; + const orderStruct = addOrderLog.order as any as Order; + const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + const order = ownerProfile.orders.get(addOrderLog.orderHash.toLowerCase()); + if (!order) { + ownerProfile.orders.set(addOrderLog.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens), + consumedTakeOrders: [], + }); + } else { + order.active = true; + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(addOrderLog.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens), + consumedTakeOrders: [], + }); + orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(addOrderLog.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens), + consumedTakeOrders: [], + }); + const ownerProfileMap: OwnersProfileMap = new Map(); + ownerProfileMap.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); + } + } +} + +/** + * Get a map of per owner orders per orderbook + * @param ordersDetails - Order details queried from subgraph + */ +export function handleRemoveOrders( + orderbook: string, + removeOrders: OrderArgsLog[], + orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, +) { + orderbook = orderbook.toLowerCase(); + for (let i = 0; i < removeOrders.length; i++) { + const removeOrderLog = removeOrders[i]; + const orderStruct = removeOrderLog.order as any as Order; + const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + const order = ownerProfile.orders.get(removeOrderLog.orderHash.toLowerCase()); + if (order) order.active = false; + } + } + } +} + +// /** +// * Builds and bundles orders which their details are queried from a orderbook subgraph +// * @param ordersDetails - Orders details queried from subgraph +// * @param _shuffle - To shuffle the bundled order array at the end +// * @param _bundle = If orders should be bundled based on token pair +// * @returns Array of bundled take orders +// */ +// export const bundleOrders = async( +// orderbook: string, +// addOrders: WatchOrderArgs[], +// viemClient: ViemClient, +// orderbooksOwnersOrdersMap: OrderbookOwnerMap, +// ) => { +// orderbook = orderbook.toLowerCase(); +// const orderbookOwnersOrdersMap = orderbooksOwnersOrdersMap.get(orderbook); +// if (orderbookOwnersOrdersMap) { +// for (const [k, v] of orderbookOwnersOrdersMap.entries()) { +// const addOrder = addOrders.find(e => e.sender.toLowerCase() === k.toLowerCase()); +// if (addOrder) { +// const orderPair = v.processed.find(e => e.takeOrders[0].id.toLowerCase() === addOrder.orderHash.toLowerCase()); +// if () { + +// } +// } else { + +// } +// } +// } +// const tokenAddresses: string[] = []; +// for (let i = 0; i < addOrders.length; i++) { +// const addOrder = addOrders[i]; +// for (let j = 0; j < addOrder.order.validOutputs.length; j++) { +// const token = addOrder.order.validOutputs[j].token.toLowerCase(); +// if (!tokenAddresses.includes(token)) tokenAddresses.push(token); +// } +// for (let j = 0; j < addOrder.order.validInputs.length; j++) { +// const token = addOrder.order.validInputs[j].token.toLowerCase(); +// if (!tokenAddresses.includes(token)) tokenAddresses.push(token); +// } +// } +// const symbols = await getBatchTokenSymbol(tokenAddresses, viemClient); +// for (let j = 0; j < addOrder.order.validOutputs.length; j++) { +// const _output = addOrder.order.validOutputs[j]; +// const _outputSymbolIndex = tokenAddresses.findIndex( +// (v: any) => v === _output.token.toLowerCase(), +// ); +// const _outputSymbol = _outputSymbolIndex > -1 ? _outputSymbolIndex : "UnknownSymbol" + +// for (let k = 0; k < addOrder.order.validInputs.length; k++) { +// const _input = addOrder.order.validInputs[k]; +// const _inputSymbolIndex = tokenAddresses.findIndex( +// (v: any) => v === _output.token.toLowerCase(), +// ); +// const _inputSymbol = _inputSymbolIndex > -1 ? _inputSymbolIndex : "UnknownSymbol" + +// if (_output.token.toLowerCase() !== _input.token.toLowerCase()) { +// if (!bundledOrders[orderbook]) { +// bundledOrders[orderbook] = []; +// } +// const pair = bundledOrders[orderbook].find( +// (v) => +// v.sellToken === _output.token.toLowerCase() && +// v.buyToken === _input.token.toLowerCase(), +// ); +// if (pair && _bundle) +// pair.takeOrders.push({ +// id: orderDetails.orderHash, +// active: true, +// takeOrder: { +// order: addOrder.order, +// inputIOIndex: k, +// outputIOIndex: j, +// signedContext: [], +// }, +// }); +// else +// bundledOrders[orderbook].push({ +// orderbook, +// buyToken: _input.token.toLowerCase(), +// buyTokenSymbol: _inputSymbol, +// buyTokenDecimals: _input.decimals, +// sellToken: _output.token.toLowerCase(), +// sellTokenSymbol: _outputSymbol, +// sellTokenDecimals: _output.decimals, +// takeOrders: [ +// { +// id: orderDetails.orderHash, +// active: true, +// takeOrder: { +// order: addOrder.order, +// inputIOIndex: k, +// outputIOIndex: j, +// signedContext: [], +// }, +// }, +// ], +// }); +// } +// } +// }} +// }; diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index ed704212..8f4b6d7d 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -29,6 +29,8 @@ const { rainterpreterNPE2Deploy, rainterpreterStoreNPE2Deploy, } = require("../utils"); +const { getOrderbookOwnersProfileMapFromSg } = require("../../src/sg"); +const { prepareRoundProcessingOrders } = require("../../src/utils"); // run tests on each network in the provided data for (let i = 0; i < testData.length; i++) { @@ -74,7 +76,7 @@ for (let i = 0; i < testData.length; i++) { for (let j = 0; j < rpVersions.length; j++) { const rpVersion = rpVersions[j]; - it.only(`should clear orders successfully using route processor v${rpVersion}`, async function () { + it(`should clear orders successfully using route processor v${rpVersion}`, async function () { config.rpc = [rpc]; const viemClient = await viem.getPublicClient(); const dataFetcher = await getDataFetcher(config, liquidityProviders, false); @@ -149,7 +151,7 @@ for (let i = 0; i < testData.length; i++) { // the deployed orders in format of a sg query. // all orders have WETH as output and other specified // tokens as input - const orders = []; + let orders = []; for (let i = 1; i < tokens.length; i++) { const depositConfigStruct = { token: tokens[i].address, @@ -249,6 +251,9 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; + orders = prepareRoundProcessingOrders( + await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + ); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders @@ -329,7 +334,7 @@ for (let i = 0; i < testData.length; i++) { testSpan.end(); }); - it.only("should clear orders successfully using inter-orderbook", async function () { + it("should clear orders successfully using inter-orderbook", async function () { config.rpc = [rpc]; const viemClient = await viem.getPublicClient(); const dataFetcher = await getDataFetcher(config, liquidityProviders, false); @@ -413,7 +418,7 @@ for (let i = 0; i < testData.length; i++) { // the deployed orders in format of a sg query. // all orders have WETH as output and other specified // tokens as input - const orders = []; + let orders = []; for (let i = 1; i < tokens.length; i++) { const depositConfigStruct1 = { token: tokens[i].address, @@ -579,6 +584,9 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; + orders = prepareRoundProcessingOrders( + await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + ); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders @@ -754,7 +762,7 @@ for (let i = 0; i < testData.length; i++) { // the deployed orders in format of a sg query. // all orders have WETH as output and other specified // tokens as input - const orders = []; + let orders = []; for (let i = 1; i < tokens.length; i++) { const depositConfigStruct1 = { token: tokens[i].address, @@ -930,6 +938,10 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; + orders = prepareRoundProcessingOrders( + await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + ); + console.log(orders); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders diff --git a/test/sg.test.js b/test/sg.test.js index 1daabaeb..6d189f64 100644 --- a/test/sg.test.js +++ b/test/sg.test.js @@ -173,30 +173,18 @@ describe("Test read subgraph", async function () { assert.deepEqual(result.availableSgs, ["url2"]); }); - it("should return correct orders details", async function () { + it.only("should return correct orders details", async function () { const sgsUrls = ["url1", "url2"]; const mockSgResultOk = [ { status: "fulfilled", reason: undefined, - value: { - data: { - data: { - orders: ["order1", "order2"], - }, - }, - }, + value: ["order1", "order2"], }, { status: "fulfilled", reason: undefined, - value: { - data: { - data: { - orders: ["order3", "order4"], - }, - }, - }, + value: ["order3", "order4"], }, ]; let result; @@ -211,24 +199,12 @@ describe("Test read subgraph", async function () { { status: "rejected", reason: undefined, - value: { - data: { - data: { - orders: ["order1", "order2"], - }, - }, - }, + value: ["order1", "order2"], }, { status: "rejected", reason: undefined, - value: { - data: { - data: { - orders: ["order3", "order4"], - }, - }, - }, + value: ["order3", "order4"], }, ]; try { @@ -242,13 +218,7 @@ describe("Test read subgraph", async function () { { status: "fulfilled", reason: undefined, - value: { - data: { - data: { - orders: ["order1", "order2"], - }, - }, - }, + value: ["order1", "order2"], }, { status: "rejected", From ed00cba9fd855ab14e808b8de9c6f86eb1f8bde8 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 25 Oct 2024 05:11:08 +0000 Subject: [PATCH 02/32] update --- src/cli.ts | 138 ++++++++++++++++--------------------------- src/processOrders.ts | 3 - src/utils.ts | 19 ++++-- src/watcher.ts | 125 +++++++-------------------------------- test/e2e/e2e.test.js | 26 ++++---- test/sg.test.js | 2 +- 6 files changed, 100 insertions(+), 213 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 531beb5c..04585815 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -246,85 +246,47 @@ export const arbRound = async ( ) => { return await tracer.startActiveSpan("process-orders", {}, roundCtx, async (span) => { const ctx = trace.setSpan(context.active(), span); - // let ordersDetails; options; try { - // try { - // ordersDetails = await getOrderDetails( - // options.subgraph, - // { - // orderHash: options.orderHash, - // orderOwner: options.orderOwner, - // orderbook: options.orderbookAddress, - // }, - // span, - // config.timeout, - // ); - // if (!ordersDetails.length) { - // span.setStatus({ code: SpanStatusCode.OK, message: "found no orders" }); - // span.end(); - // return { txs: [], foundOpp: false, avgGasCost: undefined }; - // } - // } catch (e: any) { - // const snapshot = errorSnapshot("", e); - // span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); - // span.recordException(e); - // span.setAttribute("didClear", false); - // span.setAttribute("foundOpp", false); - // span.end(); - // return { txs: [], foundOpp: false, avgGasCost: undefined }; - // } - - try { - let txs; - let foundOpp = false; - const { reports = [], avgGasCost = undefined } = await clear( - config, - bundledOrders, - tracer, - ctx, - ); - if (reports && reports.length) { - txs = reports.map((v) => v.txUrl).filter((v) => !!v); - if (txs.length) { - foundOpp = true; - span.setAttribute("txUrls", txs); - span.setAttribute("didClear", true); - span.setAttribute("foundOpp", true); - } else if ( - reports.some((v) => v.status === ProcessPairReportStatus.FoundOpportunity) - ) { - foundOpp = true; - span.setAttribute("foundOpp", true); - } - } else { - span.setAttribute("didClear", false); - } - if (avgGasCost) { - span.setAttribute("avgGasCost", avgGasCost.toString()); - } - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - return { txs, foundOpp, avgGasCost }; - } catch (e: any) { - if (e?.startsWith?.("Failed to batch quote orders")) { - span.setAttribute("severity", ErrorSeverity.LOW); - span.setStatus({ code: SpanStatusCode.ERROR, message: e }); - } else { - const snapshot = errorSnapshot("Unexpected error occured", e); - span.setAttribute("severity", ErrorSeverity.HIGH); - span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); + let txs; + let foundOpp = false; + const { reports = [], avgGasCost = undefined } = await clear( + config, + bundledOrders, + tracer, + ctx, + ); + if (reports && reports.length) { + txs = reports.map((v) => v.txUrl).filter((v) => !!v); + if (txs.length) { + foundOpp = true; + span.setAttribute("txUrls", txs); + span.setAttribute("didClear", true); + span.setAttribute("foundOpp", true); + } else if ( + reports.some((v) => v.status === ProcessPairReportStatus.FoundOpportunity) + ) { + foundOpp = true; + span.setAttribute("foundOpp", true); } - span.recordException(e); + } else { span.setAttribute("didClear", false); - span.setAttribute("foundOpp", false); - span.end(); - return { txs: [], foundOpp: false, avgGasCost: undefined }; } + if (avgGasCost) { + span.setAttribute("avgGasCost", avgGasCost.toString()); + } + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return { txs, foundOpp, avgGasCost }; } catch (e: any) { - const snapshot = errorSnapshot("Unexpected error occured", e); - span.setAttribute("severity", ErrorSeverity.HIGH); - span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); + if (e?.startsWith?.("Failed to batch quote orders")) { + span.setAttribute("severity", ErrorSeverity.LOW); + span.setStatus({ code: SpanStatusCode.ERROR, message: e }); + } else { + const snapshot = errorSnapshot("Unexpected error occured", e); + span.setAttribute("severity", ErrorSeverity.HIGH); + span.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); + } span.recordException(e); span.setAttribute("didClear", false); span.setAttribute("foundOpp", false); @@ -403,7 +365,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? } } const tokens = getOrdersTokens(ordersDetails); - options.tokens = tokens; + options.tokens = [...tokens]; // get config const config = await getConfig( @@ -497,12 +459,7 @@ export const main = async (argv: any, version?: string) => { orderbooksOwnersProfileMap.forEach((_, ob) => { obs.push(ob.toLowerCase()); }); - const orderbooksUnwatchers = watchAllOrderbooks( - obs, - config.watchClient, - watchedOrderbooksOrders, - ); - orderbooksUnwatchers; + watchAllOrderbooks(obs, config.watchClient, watchedOrderbooksOrders); const day = 24 * 60 * 60 * 1000; let lastGasReset = Date.now() + day; @@ -576,13 +533,9 @@ export const main = async (argv: any, version?: string) => { update = true; } try { - const bundledOrders = prepareRoundProcessingOrders(orderbooksOwnersProfileMap); - await handleNewLogs( + const bundledOrders = prepareRoundProcessingOrders( orderbooksOwnersProfileMap, - watchedOrderbooksOrders, - config.viemClient as any as ViemClient, - tokens, - (options as CliOptions).ownerProfile, + true, ); await rotateProviders(config, update); const roundResult = await arbRound( @@ -704,6 +657,19 @@ export const main = async (argv: any, version?: string) => { if (avgGasCost) { roundSpan.setAttribute("avgGasCost", ethers.utils.formatUnits(avgGasCost)); } + try { + // check for new orders + await handleNewLogs( + orderbooksOwnersProfileMap, + watchedOrderbooksOrders, + config.viemClient as any as ViemClient, + tokens, + options.ownerProfile, + roundSpan, + ); + } catch { + /**/ + } // eslint-disable-next-line no-console console.log(`Starting next round in ${roundGap / 1000} seconds...`, "\n"); roundSpan.end(); diff --git a/src/processOrders.ts b/src/processOrders.ts index 5b73c1bf..fdb8b22a 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -84,9 +84,6 @@ export const processOrders = async ( genericArb = new ethers.Contract(config.genericArbAddress, arbAbis); } - // prepare orders - // const bundledOrders = bundleOrders(ordersDetails, false, true); - // check owned vaults and top them up if necessary await tracer.startActiveSpan("handle-owned-vaults", {}, ctx, async (span) => { try { diff --git a/src/utils.ts b/src/utils.ts index 3029de7c..6f31c596 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1309,6 +1309,7 @@ export async function getTokenSymbol(address: string, viemClient: ViemClient): P export function prepareRoundProcessingOrders( orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + shuffle = true, ): BundledOrders[][] { const result: BundledOrders[][] = []; for (const [orderbook, ownersProfileMap] of orderbooksOwnersProfileMap) { @@ -1338,8 +1339,20 @@ export function prepareRoundProcessingOrders( } } } + if (shuffle) { + // shuffle orders + for (const bundledOrders of orderbookBundledOrders) { + shuffleArray(bundledOrders.takeOrders); + } + // shuffle pairs + shuffleArray(orderbookBundledOrders); + } result.push(orderbookBundledOrders); } + if (shuffle) { + // shuffle orderbooks + shuffleArray(result); + } return result; } @@ -1353,11 +1366,7 @@ function gatherPairs( const bundleOrder = bundledOrders.find( (v) => v.buyToken.toLowerCase() === pair.buyToken.toLowerCase() && - v.buyTokenDecimals === pair.buyTokenDecimals && - v.buyTokenSymbol === pair.buyTokenSymbol && - v.sellToken.toLowerCase() === pair.sellToken.toLowerCase() && - v.sellTokenDecimals === pair.sellTokenDecimals && - v.sellTokenSymbol === pair.sellTokenSymbol, + v.sellToken.toLowerCase() === pair.sellToken.toLowerCase(), ); if (bundleOrder) { if ( diff --git a/src/watcher.ts b/src/watcher.ts index 51bd333a..fd889023 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -10,6 +10,7 @@ import { OrderbooksOwnersProfileMap, } from "./types"; import { hexlify } from "ethers/lib/utils"; +import { Span } from "@opentelemetry/api"; type OrderEventLog = { sender: `0x${string}`; @@ -146,6 +147,7 @@ export async function handleNewLogs( viemClient: ViemClient, tokens: TokenDetails[], ownerLimits?: Record, + span?: Span, ) { for (const ob in watchedOrderbooksOrders) { const watchedOrderbookLogs = watchedOrderbooksOrders[ob]; @@ -156,11 +158,13 @@ export async function handleNewLogs( viemClient, tokens, ownerLimits, + span, ); handleRemoveOrders( ob, watchedOrderbookLogs.removeOrders.splice(0), orderbooksOwnersProfileMap, + span, ); } } @@ -176,11 +180,16 @@ export async function handleAddOrders( viemClient: ViemClient, tokens: TokenDetails[], ownerLimits?: Record, + span?: Span, ) { orderbook = orderbook.toLowerCase(); + span?.setAttribute( + "details.newOrders", + addOrders.map((v) => v.orderHash), + ); for (let i = 0; i < addOrders.length; i++) { const addOrderLog = addOrders[i]; - const orderStruct = addOrderLog.order as any as Order; + const orderStruct = addOrderLog.order; const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); if (orderbookOwnerProfileItem) { const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); @@ -235,120 +244,26 @@ export function handleRemoveOrders( orderbook: string, removeOrders: OrderArgsLog[], orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, + span?: Span, ) { orderbook = orderbook.toLowerCase(); + span?.setAttribute( + "details.removedOrders", + removeOrders.map((v) => v.orderHash), + ); for (let i = 0; i < removeOrders.length; i++) { const removeOrderLog = removeOrders[i]; - const orderStruct = removeOrderLog.order as any as Order; + const orderStruct = removeOrderLog.order; const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); if (orderbookOwnerProfileItem) { const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); if (ownerProfile) { const order = ownerProfile.orders.get(removeOrderLog.orderHash.toLowerCase()); - if (order) order.active = false; + if (order) { + order.active = false; + order.takeOrders.push(...order.consumedTakeOrders.splice(0)); + } } } } } - -// /** -// * Builds and bundles orders which their details are queried from a orderbook subgraph -// * @param ordersDetails - Orders details queried from subgraph -// * @param _shuffle - To shuffle the bundled order array at the end -// * @param _bundle = If orders should be bundled based on token pair -// * @returns Array of bundled take orders -// */ -// export const bundleOrders = async( -// orderbook: string, -// addOrders: WatchOrderArgs[], -// viemClient: ViemClient, -// orderbooksOwnersOrdersMap: OrderbookOwnerMap, -// ) => { -// orderbook = orderbook.toLowerCase(); -// const orderbookOwnersOrdersMap = orderbooksOwnersOrdersMap.get(orderbook); -// if (orderbookOwnersOrdersMap) { -// for (const [k, v] of orderbookOwnersOrdersMap.entries()) { -// const addOrder = addOrders.find(e => e.sender.toLowerCase() === k.toLowerCase()); -// if (addOrder) { -// const orderPair = v.processed.find(e => e.takeOrders[0].id.toLowerCase() === addOrder.orderHash.toLowerCase()); -// if () { - -// } -// } else { - -// } -// } -// } -// const tokenAddresses: string[] = []; -// for (let i = 0; i < addOrders.length; i++) { -// const addOrder = addOrders[i]; -// for (let j = 0; j < addOrder.order.validOutputs.length; j++) { -// const token = addOrder.order.validOutputs[j].token.toLowerCase(); -// if (!tokenAddresses.includes(token)) tokenAddresses.push(token); -// } -// for (let j = 0; j < addOrder.order.validInputs.length; j++) { -// const token = addOrder.order.validInputs[j].token.toLowerCase(); -// if (!tokenAddresses.includes(token)) tokenAddresses.push(token); -// } -// } -// const symbols = await getBatchTokenSymbol(tokenAddresses, viemClient); -// for (let j = 0; j < addOrder.order.validOutputs.length; j++) { -// const _output = addOrder.order.validOutputs[j]; -// const _outputSymbolIndex = tokenAddresses.findIndex( -// (v: any) => v === _output.token.toLowerCase(), -// ); -// const _outputSymbol = _outputSymbolIndex > -1 ? _outputSymbolIndex : "UnknownSymbol" - -// for (let k = 0; k < addOrder.order.validInputs.length; k++) { -// const _input = addOrder.order.validInputs[k]; -// const _inputSymbolIndex = tokenAddresses.findIndex( -// (v: any) => v === _output.token.toLowerCase(), -// ); -// const _inputSymbol = _inputSymbolIndex > -1 ? _inputSymbolIndex : "UnknownSymbol" - -// if (_output.token.toLowerCase() !== _input.token.toLowerCase()) { -// if (!bundledOrders[orderbook]) { -// bundledOrders[orderbook] = []; -// } -// const pair = bundledOrders[orderbook].find( -// (v) => -// v.sellToken === _output.token.toLowerCase() && -// v.buyToken === _input.token.toLowerCase(), -// ); -// if (pair && _bundle) -// pair.takeOrders.push({ -// id: orderDetails.orderHash, -// active: true, -// takeOrder: { -// order: addOrder.order, -// inputIOIndex: k, -// outputIOIndex: j, -// signedContext: [], -// }, -// }); -// else -// bundledOrders[orderbook].push({ -// orderbook, -// buyToken: _input.token.toLowerCase(), -// buyTokenSymbol: _inputSymbol, -// buyTokenDecimals: _input.decimals, -// sellToken: _output.token.toLowerCase(), -// sellTokenSymbol: _outputSymbol, -// sellTokenDecimals: _output.decimals, -// takeOrders: [ -// { -// id: orderDetails.orderHash, -// active: true, -// takeOrder: { -// order: addOrder.order, -// inputIOIndex: k, -// outputIOIndex: j, -// signedContext: [], -// }, -// }, -// ], -// }); -// } -// } -// }} -// }; diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index 8f4b6d7d..515d8a77 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -11,9 +11,11 @@ const { trace, context } = require("@opentelemetry/api"); const { publicActions, walletActions } = require("viem"); const ERC20Artifact = require("../abis/ERC20Upgradeable.json"); const { abi: orderbookAbi } = require("../abis/OrderBook.json"); +const { prepareRoundProcessingOrders } = require("../../src/utils"); const helpers = require("@nomicfoundation/hardhat-network-helpers"); const { ProcessPairReportStatus } = require("../../src/processOrders"); const { getChainConfig, getDataFetcher } = require("../../src/config"); +const { getOrderbookOwnersProfileMapFromSg } = require("../../src/sg"); const { OTLPTraceExporter } = require("@opentelemetry/exporter-trace-otlp-http"); const { SEMRESATTRS_SERVICE_NAME } = require("@opentelemetry/semantic-conventions"); const { BasicTracerProvider, BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base"); @@ -29,8 +31,6 @@ const { rainterpreterNPE2Deploy, rainterpreterStoreNPE2Deploy, } = require("../utils"); -const { getOrderbookOwnersProfileMapFromSg } = require("../../src/sg"); -const { prepareRoundProcessingOrders } = require("../../src/utils"); // run tests on each network in the provided data for (let i = 0; i < testData.length; i++) { @@ -253,6 +253,7 @@ for (let i = 0; i < testData.length; i++) { config.quoteRpc = [mockServer.url + "/rpc"]; orders = prepareRoundProcessingOrders( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + false, ); const { reports } = await clear(config, orders, tracer, ctx); @@ -586,6 +587,7 @@ for (let i = 0; i < testData.length; i++) { config.quoteRpc = [mockServer.url + "/rpc"]; orders = prepareRoundProcessingOrders( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + false, ); const { reports } = await clear(config, orders, tracer, ctx); @@ -680,7 +682,7 @@ for (let i = 0; i < testData.length; i++) { testSpan.end(); }); - it.only("should clear orders successfully using intra-orderbook", async function () { + it("should clear orders successfully using intra-orderbook", async function () { config.rpc = [rpc]; const viemClient = await viem.getPublicClient(); const dataFetcher = await getDataFetcher(config, liquidityProviders, false); @@ -881,24 +883,22 @@ for (let i = 0; i < testData.length; i++) { } // mock quote responses + const t0 = []; + for (let i = 0; i < tokens.length - 1; i++) { + t0.push(tokens[0]); + } await mockServer .forPost("/rpc") .once() + // .withBodyIncluding(owners[0].address.substring(2).toLowerCase()) .thenSendJsonRpcResult( encodeQuoteResponse([ - ...tokens.slice(1).flatMap((v) => [ + ...[tokens[1], ...t0, ...tokens.slice(2)].flatMap((v) => [ [ true, // success v.depositAmount.mul("1" + "0".repeat(18 - v.decimals)), //maxout ethers.constants.Zero, // ratio ], - [ - true, - tokens[0].depositAmount.mul( - "1" + "0".repeat(18 - tokens[0].decimals), - ), - ethers.constants.Zero, - ], ]), ]), ); @@ -940,8 +940,8 @@ for (let i = 0; i < testData.length; i++) { config.quoteRpc = [mockServer.url + "/rpc"]; orders = prepareRoundProcessingOrders( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), + false, ); - console.log(orders); const { reports } = await clear(config, orders, tracer, ctx); // should have cleared correct number of orders @@ -955,7 +955,7 @@ for (let i = 0; i < testData.length; i++) { let gasSpent = ethers.constants.Zero; let inputProfit = ethers.constants.Zero; for (let i = 0; i < reports.length; i++) { - if (i % 2 !== 0) continue; + if (reports[i].status !== ProcessPairReportStatus.FoundOpportunity) continue; assert.equal(reports[i].status, ProcessPairReportStatus.FoundOpportunity); assert.equal(reports[i].clearedOrders.length, 1); diff --git a/test/sg.test.js b/test/sg.test.js index 6d189f64..aec25631 100644 --- a/test/sg.test.js +++ b/test/sg.test.js @@ -173,7 +173,7 @@ describe("Test read subgraph", async function () { assert.deepEqual(result.availableSgs, ["url2"]); }); - it.only("should return correct orders details", async function () { + it("should return correct orders details", async function () { const sgsUrls = ["url1", "url2"]; const mockSgResultOk = [ { From f7c71b0444b63f764ffcae30a6d45e1dec1da26e Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 25 Oct 2024 20:42:20 +0000 Subject: [PATCH 03/32] update --- src/cli.ts | 10 +- src/order.ts | 260 +++++++++++++++++++++++++++++++++++++++++++ src/processOrders.ts | 1 - src/sg.ts | 153 +------------------------ src/utils.ts | 120 +------------------- src/watcher.ts | 14 ++- test/orders.test.js | 113 +++++++++++++++++++ 7 files changed, 389 insertions(+), 282 deletions(-) create mode 100644 src/order.ts diff --git a/src/cli.ts b/src/cli.ts index 04585815..53f64222 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,18 +1,21 @@ import { config } from "dotenv"; +import { SgOrder } from "./query"; import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; -import { sleep, getOrdersTokens, prepareRoundProcessingOrders } from "./utils"; +import { sleep, getOrdersTokens } from "./utils"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; -import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; +import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { handleNewLogs, watchAllOrderbooks, WatchedOrderbookOrders } from "./watcher"; +import { getOrderbookOwnersProfileMapFromSg, prepareRoundProcessingOrders } from "./order"; import { manageAccounts, rotateProviders, sweepToMainWallet, sweepToEth } from "./account"; import { diag, @@ -28,9 +31,6 @@ import { ConsoleSpanExporter, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; -import { handleNewLogs, watchAllOrderbooks, WatchedOrderbookOrders } from "./watcher"; -import { SgOrder } from "./query"; -import { getOrderbookOwnersProfileMapFromSg } from "./sg"; config(); diff --git a/src/order.ts b/src/order.ts new file mode 100644 index 00000000..13750fe8 --- /dev/null +++ b/src/order.ts @@ -0,0 +1,260 @@ +import { SgOrder } from "./query"; +import { toOrder } from "./watcher"; +import { shuffleArray } from "./utils"; +import { erc20Abi, OrderV3 } from "./abis"; +import { decodeAbiParameters, parseAbi, parseAbiParameters } from "viem"; +import { + Pair, + Order, + ViemClient, + TokenDetails, + BundledOrders, + OrdersProfileMap, + OwnersProfileMap, + OrderbooksOwnersProfileMap, +} from "./types"; + +export async function getPairs( + orderStruct: Order, + viemClient: ViemClient, + tokens: TokenDetails[], + orderDetails?: SgOrder, +): Promise { + const pairs: Pair[] = []; + for (let j = 0; j < orderStruct.validOutputs.length; j++) { + const _output = orderStruct.validOutputs[j]; + let _outputSymbol = orderDetails?.outputs?.find( + (v) => v.token.address.toLowerCase() === _output.token.toLowerCase(), + )?.token?.symbol; + if (!_outputSymbol) { + const symbol = tokens.find( + (v) => v.address.toLowerCase() === _output.token.toLowerCase(), + )?.symbol; + if (!symbol) { + _outputSymbol = await getTokenSymbol(_output.token, viemClient); + } else { + _outputSymbol = symbol; + } + } else { + if (!tokens.find((v) => v.address.toLowerCase() === _output.token.toLowerCase())) { + tokens.push({ + address: _output.token.toLowerCase(), + symbol: _outputSymbol, + decimals: _output.decimals, + }); + } + } + + for (let k = 0; k < orderStruct.validInputs.length; k++) { + const _input = orderStruct.validInputs[k]; + let _inputSymbol = orderDetails?.inputs?.find( + (v) => v.token.address.toLowerCase() === _input.token.toLowerCase(), + )?.token?.symbol; + if (!_inputSymbol) { + const symbol = tokens.find( + (v) => v.address.toLowerCase() === _input.token.toLowerCase(), + )?.symbol; + if (!symbol) { + _inputSymbol = await getTokenSymbol(_input.token, viemClient); + } else { + _inputSymbol = symbol; + } + } else { + if (!tokens.find((v) => v.address.toLowerCase() === _input.token.toLowerCase())) { + tokens.push({ + address: _input.token.toLowerCase(), + symbol: _inputSymbol, + decimals: _input.decimals, + }); + } + } + + if (_input.token.toLowerCase() !== _output.token.toLowerCase()) + pairs.push({ + buyToken: _input.token.toLowerCase(), + buyTokenSymbol: _inputSymbol, + buyTokenDecimals: _input.decimals, + sellToken: _output.token.toLowerCase(), + sellTokenSymbol: _outputSymbol, + sellTokenDecimals: _output.decimals, + takeOrder: { + order: orderStruct, + inputIOIndex: k, + outputIOIndex: j, + signedContext: [], + }, + }); + } + } + return pairs; +} +/** + * Get a map of per owner orders per orderbook + * @param ordersDetails - Order details queried from subgraph + */ +export async function getOrderbookOwnersProfileMapFromSg( + ordersDetails: SgOrder[], + viemClient: ViemClient, + tokens: TokenDetails[], + ownerLimits?: Record, +): Promise { + const orderbookOwnersProfileMap: OrderbooksOwnersProfileMap = new Map(); + for (let i = 0; i < ordersDetails.length; i++) { + const orderDetails = ordersDetails[i]; + const orderbook = orderDetails.orderbook.id.toLowerCase(); + const orderStruct = toOrder( + decodeAbiParameters( + parseAbiParameters(OrderV3), + orderDetails.orderBytes as `0x${string}`, + )[0], + ); + const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + if (!ownerProfile.orders.has(orderDetails.orderHash.toLowerCase())) { + ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + consumedTakeOrders: [], + }); + const ownerProfileMap: OwnersProfileMap = new Map(); + ownerProfileMap.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + orders: ordersProfileMap, + }); + orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); + } + } + return orderbookOwnersProfileMap; +} + +/** + * Get token symbol + * @param address - The address of token + * @param viemClient - The viem client + */ +export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { + try { + return await viemClient.readContract({ + address: address as `0x${string}`, + abi: parseAbi(erc20Abi), + functionName: "symbol", + }); + } catch (error) { + return "Unknownsymbol"; + } +} + +export function prepareRoundProcessingOrders( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + shuffle = true, +): BundledOrders[][] { + const result: BundledOrders[][] = []; + for (const [orderbook, ownersProfileMap] of orderbooksOwnersProfileMap) { + const orderbookBundledOrders: BundledOrders[] = []; + for (const [, ownerProfile] of ownersProfileMap) { + let consumedLimit = ownerProfile.limit; + const activeOrdersProfiles = Array.from(ownerProfile.orders).filter((v) => v[1].active); + const remainingOrdersPairs = activeOrdersProfiles.filter( + (v) => v[1].takeOrders.length > 0, + ); + if (remainingOrdersPairs.length === 0) { + for (const [, orderProfile] of activeOrdersProfiles) { + orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); + } + for (const [orderHash, orderProfile] of activeOrdersProfiles) { + const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); + consumedLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + } + } else { + for (const [orderHash, orderProfile] of remainingOrdersPairs) { + const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); + consumedLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + } + } + } + if (shuffle) { + // shuffle orders + for (const bundledOrders of orderbookBundledOrders) { + shuffleArray(bundledOrders.takeOrders); + } + // shuffle pairs + shuffleArray(orderbookBundledOrders); + } + result.push(orderbookBundledOrders); + } + if (shuffle) { + // shuffle orderbooks + shuffleArray(result); + } + return result; +} + +function gatherPairs( + orderbook: string, + orderHash: string, + pairs: Pair[], + bundledOrders: BundledOrders[], +) { + for (const pair of pairs) { + const bundleOrder = bundledOrders.find( + (v) => + v.buyToken.toLowerCase() === pair.buyToken.toLowerCase() && + v.sellToken.toLowerCase() === pair.sellToken.toLowerCase(), + ); + if (bundleOrder) { + if ( + !bundleOrder.takeOrders.find((v) => v.id.toLowerCase() === orderHash.toLowerCase()) + ) { + bundleOrder.takeOrders.push({ + id: orderHash, + takeOrder: pair.takeOrder, + }); + } + } else { + bundledOrders.push({ + orderbook, + buyToken: pair.buyToken, + buyTokenDecimals: pair.buyTokenDecimals, + buyTokenSymbol: pair.buyTokenSymbol, + sellToken: pair.sellToken, + sellTokenDecimals: pair.sellTokenDecimals, + sellTokenSymbol: pair.sellTokenSymbol, + takeOrders: [ + { + id: orderHash, + takeOrder: pair.takeOrder, + }, + ], + }); + } + } +} diff --git a/src/processOrders.ts b/src/processOrders.ts index fdb8b22a..27aae63d 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -27,7 +27,6 @@ import { getEthPrice, quoteOrders, routeExists, - // bundleOrders, PoolBlackList, getMarketQuote, getTotalIncome, diff --git a/src/sg.ts b/src/sg.ts index aa745ddb..16dab77b 100644 --- a/src/sg.ts +++ b/src/sg.ts @@ -1,20 +1,7 @@ import axios from "axios"; import { ErrorSeverity } from "./error"; import { Span } from "@opentelemetry/api"; -import { orderbooksQuery, SgOrder } from "./query"; -import { getTokenSymbol } from "./utils"; -import { - Order, - OrderbooksOwnersProfileMap, - OrdersProfileMap, - OwnersProfileMap, - Pair, - TokenDetails, - ViemClient, -} from "./types"; -import { OrderV3 } from "./abis"; -import { decodeAbiParameters, parseAbiParameters } from "viem"; -import { toOrder } from "./watcher"; +import { orderbooksQuery } from "./query"; /** * Checks a subgraph health status and records the result in an object or throws @@ -151,141 +138,3 @@ export async function getSgOrderbooks(url: string): Promise { throw msg.join("\n"); } } - -export async function getPairs( - orderStruct: Order, - viemClient: ViemClient, - tokens: TokenDetails[], - orderDetails?: SgOrder, -): Promise { - const pairs: Pair[] = []; - for (let j = 0; j < orderStruct.validOutputs.length; j++) { - const _output = orderStruct.validOutputs[j]; - let _outputSymbol = orderDetails?.outputs?.find( - (v) => v.token.address.toLowerCase() === _output.token.toLowerCase(), - )?.token?.symbol; - if (!_outputSymbol) { - const symbol = tokens.find( - (v) => v.address.toLowerCase() === _output.token.toLowerCase(), - )?.symbol; - if (!symbol) { - _outputSymbol = await getTokenSymbol(_output.token, viemClient); - } else { - _outputSymbol = symbol; - } - } else { - if (!tokens.find((v) => v.address.toLowerCase() === _output.token.toLowerCase())) { - tokens.push({ - address: _output.token.toLowerCase(), - symbol: _outputSymbol, - decimals: _output.decimals, - }); - } - } - - for (let k = 0; k < orderStruct.validInputs.length; k++) { - const _input = orderStruct.validInputs[k]; - let _inputSymbol = orderDetails?.inputs?.find( - (v) => v.token.address.toLowerCase() === _input.token.toLowerCase(), - )?.token?.symbol; - if (!_inputSymbol) { - const symbol = tokens.find( - (v) => v.address.toLowerCase() === _input.token.toLowerCase(), - )?.symbol; - if (!symbol) { - _inputSymbol = await getTokenSymbol(_input.token, viemClient); - } else { - _inputSymbol = symbol; - } - } else { - if (!tokens.find((v) => v.address.toLowerCase() === _input.token.toLowerCase())) { - tokens.push({ - address: _input.token.toLowerCase(), - symbol: _inputSymbol, - decimals: _input.decimals, - }); - } - } - - if (_input.token.toLowerCase() !== _output.token.toLowerCase()) - pairs.push({ - buyToken: _input.token.toLowerCase(), - buyTokenSymbol: _inputSymbol, - buyTokenDecimals: _input.decimals, - sellToken: _output.token.toLowerCase(), - sellTokenSymbol: _outputSymbol, - sellTokenDecimals: _output.decimals, - takeOrder: { - order: orderStruct, - inputIOIndex: k, - outputIOIndex: j, - signedContext: [], - }, - }); - } - } - return pairs; -} -/** - * Get a map of per owner orders per orderbook - * @param ordersDetails - Order details queried from subgraph - */ -export async function getOrderbookOwnersProfileMapFromSg( - ordersDetails: SgOrder[], - viemClient: ViemClient, - tokens: TokenDetails[], - ownerLimits?: Record, -): Promise { - const orderbookOwnersProfileMap: OrderbooksOwnersProfileMap = new Map(); - for (let i = 0; i < ordersDetails.length; i++) { - const orderDetails = ordersDetails[i]; - const orderbook = orderDetails.orderbook.id.toLowerCase(); - const orderStruct = toOrder( - decodeAbiParameters( - parseAbiParameters(OrderV3), - orderDetails.orderBytes as `0x${string}`, - )[0], - ); - const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); - if (orderbookOwnerProfileItem) { - const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); - if (ownerProfile) { - if (!ownerProfile.orders.has(orderDetails.orderHash.toLowerCase())) { - ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), - consumedTakeOrders: [], - }); - } - } else { - const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), - consumedTakeOrders: [], - }); - orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, - orders: ordersProfileMap, - }); - } - } else { - const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), - consumedTakeOrders: [], - }); - const ownerProfileMap: OwnersProfileMap = new Map(); - ownerProfileMap.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, - orders: ordersProfileMap, - }); - orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); - } - } - return orderbookOwnersProfileMap; -} diff --git a/src/utils.ts b/src/utils.ts index 6f31c596..dcea6297 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,4 @@ +import { SgOrder } from "./query"; import { ChainId } from "sushi/chain"; import { RouteLeg } from "sushi/tines"; import { getDataFetcher } from "./config"; @@ -7,18 +8,8 @@ import { BigNumber, BigNumberish, ethers } from "ethers"; import { erc20Abi, orderbookAbi, OrderV3 } from "./abis"; import { parseAbi, PublicClient, TransactionReceipt } from "viem"; import { doQuoteTargets, QuoteTarget } from "@rainlanguage/orderbook/quote"; -import { - BotConfig, - BundledOrders, - OrderbooksOwnersProfileMap, - OwnedOrder, - Pair, - TakeOrder, - TokenDetails, - ViemClient, -} from "./types"; +import { BotConfig, BundledOrders, OwnedOrder, TakeOrder, TokenDetails } from "./types"; import { DataFetcher, DataFetcherOptions, LiquidityProviders, Router } from "sushi/router"; -import { SgOrder } from "./query"; export function RPoolFilter(pool: any) { return !BlackList.includes(pool.address) && !BlackList.includes(pool.address.toLowerCase()); @@ -1289,110 +1280,3 @@ export function getMarketQuote( }; } } - -/** - * Get token symbol - * @param address - The address of token - * @param viemClient - The viem client - */ -export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { - try { - return await viemClient.readContract({ - address: address as `0x${string}`, - abi: parseAbi(erc20Abi), - functionName: "symbol", - }); - } catch (error) { - return "Unknownsymbol"; - } -} - -export function prepareRoundProcessingOrders( - orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, - shuffle = true, -): BundledOrders[][] { - const result: BundledOrders[][] = []; - for (const [orderbook, ownersProfileMap] of orderbooksOwnersProfileMap) { - const orderbookBundledOrders: BundledOrders[] = []; - for (const [, ownerProfile] of ownersProfileMap) { - let consumedLimit = ownerProfile.limit; - const activeOrdersProfiles = Array.from(ownerProfile.orders).filter((v) => v[1].active); - const remainingOrdersPairs = activeOrdersProfiles.filter( - (v) => v[1].takeOrders.length > 0, - ); - if (remainingOrdersPairs.length === 0) { - for (const [, orderProfile] of activeOrdersProfiles) { - orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); - } - for (const [orderHash, orderProfile] of activeOrdersProfiles) { - const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); - consumedLimit -= consumingOrderPairs.length; - orderProfile.consumedTakeOrders.push(...consumingOrderPairs); - gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); - } - } else { - for (const [orderHash, orderProfile] of remainingOrdersPairs) { - const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); - consumedLimit -= consumingOrderPairs.length; - orderProfile.consumedTakeOrders.push(...consumingOrderPairs); - gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); - } - } - } - if (shuffle) { - // shuffle orders - for (const bundledOrders of orderbookBundledOrders) { - shuffleArray(bundledOrders.takeOrders); - } - // shuffle pairs - shuffleArray(orderbookBundledOrders); - } - result.push(orderbookBundledOrders); - } - if (shuffle) { - // shuffle orderbooks - shuffleArray(result); - } - return result; -} - -function gatherPairs( - orderbook: string, - orderHash: string, - pairs: Pair[], - bundledOrders: BundledOrders[], -) { - for (const pair of pairs) { - const bundleOrder = bundledOrders.find( - (v) => - v.buyToken.toLowerCase() === pair.buyToken.toLowerCase() && - v.sellToken.toLowerCase() === pair.sellToken.toLowerCase(), - ); - if (bundleOrder) { - if ( - !bundleOrder.takeOrders.find((v) => v.id.toLowerCase() === orderHash.toLowerCase()) - ) { - bundleOrder.takeOrders.push({ - id: orderHash, - takeOrder: pair.takeOrder, - }); - } - } else { - bundledOrders.push({ - orderbook, - buyToken: pair.buyToken, - buyTokenDecimals: pair.buyTokenDecimals, - buyTokenSymbol: pair.buyTokenSymbol, - sellToken: pair.sellToken, - sellTokenDecimals: pair.sellTokenDecimals, - sellTokenSymbol: pair.sellTokenSymbol, - takeOrders: [ - { - id: orderHash, - takeOrder: pair.takeOrder, - }, - ], - }); - } - } -} diff --git a/src/watcher.ts b/src/watcher.ts index fd889023..c391f30b 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -1,4 +1,6 @@ -import { getPairs } from "./sg"; +import { getPairs } from "./order"; +import { Span } from "@opentelemetry/api"; +import { hexlify } from "ethers/lib/utils"; import { orderbookAbi as abi } from "./abis"; import { parseAbi, WatchContractEventReturnType } from "viem"; import { @@ -9,8 +11,6 @@ import { OwnersProfileMap, OrderbooksOwnersProfileMap, } from "./types"; -import { hexlify } from "ethers/lib/utils"; -import { Span } from "@opentelemetry/api"; type OrderEventLog = { sender: `0x${string}`; @@ -42,7 +42,7 @@ export type OrderArgsLog = { }; export type WatchedOrderbookOrders = { addOrders: OrderArgsLog[]; removeOrders: OrderArgsLog[] }; -function toOrderArgsLog(orderLog: OrderEventLog): OrderArgsLog { +function logToOrder(orderLog: OrderEventLog): OrderArgsLog { return { sender: orderLog.sender, orderHash: orderLog.orderHash, @@ -87,11 +87,12 @@ export function watchOrderbook( address: orderbook as `0x${string}`, abi: orderbookAbi, eventName: "AddOrderV2", + pollingInterval: 30_000, onLogs: (logs) => { logs.forEach((log) => { if (log) { watchedOrderbookOrders.addOrders.push( - toOrderArgsLog(log.args as any as OrderEventLog), + logToOrder(log.args as any as OrderEventLog), ); } }); @@ -102,11 +103,12 @@ export function watchOrderbook( address: orderbook as `0x${string}`, abi: orderbookAbi, eventName: "RemoveOrderV2", + pollingInterval: 30_000, onLogs: (logs) => { logs.forEach((log) => { if (log) { watchedOrderbookOrders.removeOrders.push( - toOrderArgsLog(log.args as any as OrderEventLog), + logToOrder(log.args as any as OrderEventLog), ); } }); diff --git a/test/orders.test.js b/test/orders.test.js index 9a637016..fc34e544 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -5,6 +5,9 @@ const { ethers, viem, network } = require("hardhat"); const ERC20Artifact = require("./abis/ERC20Upgradeable.json"); const { deployOrderBookNPE2, encodeQuoteResponse } = require("./utils"); const { bundleOrders, getVaultBalance, quoteOrders, quoteSingleOrder } = require("../src/utils"); +const { toOrder } = require("../src/watcher"); +const { decodeAbiParameters, parseAbiParameters } = require("viem"); +const { getPairs, getOrderbookOwnersProfileMapFromSg } = require("../src/order"); describe("Test order details", async function () { beforeEach(() => mockServer.start(8081)); @@ -545,6 +548,116 @@ describe("Test order details", async function () { }; assert.deepEqual(orderDetails.takeOrders[0].quote, expected); }); + + it("should get order pairs", async function () { + const orderStruct = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], + ); + const result = await getPairs(orderStruct, undefined, [], order1); + const expected = [ + { + buyToken: orderStruct1.validInputs[0].token, + buyTokenSymbol: order1.inputs[0].token.symbol, + buyTokenDecimals: orderStruct1.validInputs[0].decimals, + sellToken: orderStruct1.validOutputs[0].token, + sellTokenSymbol: order1.outputs[0].token.symbol, + sellTokenDecimals: orderStruct1.validOutputs[0].decimals, + takeOrder: { + order: orderStruct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + }, + ]; + assert.deepEqual(result, expected); + }); + + it("should make orderbook owner order map", async function () { + const orderStruct1 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], + ); + const orderStruct2 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order2.orderBytes)[0], + ); + const result = await getOrderbookOwnersProfileMapFromSg( + [order1, order2], + undefined, + [], + {}, + ); + const ownerMap = new Map(); + ownerMap.set(order1.owner.toLowerCase(), { + limit: 25, + orders: new Map([ + [ + order1.orderHash.toLowerCase(), + { + active: true, + order: orderStruct1, + consumedTakeOrders: [], + takeOrders: [ + { + buyToken: orderStruct1.validInputs[0].token, + buyTokenSymbol: order1.inputs[0].token.symbol, + buyTokenDecimals: orderStruct1.validInputs[0].decimals, + sellToken: orderStruct1.validOutputs[0].token, + sellTokenSymbol: order1.outputs[0].token.symbol, + sellTokenDecimals: orderStruct1.validOutputs[0].decimals, + takeOrder: { + order: orderStruct1, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + }, + ], + }, + ], + ]), + }); + ownerMap.set(order2.owner.toLowerCase(), { + limit: 25, + orders: new Map([ + [ + order2.orderHash.toLowerCase(), + { + active: true, + order: orderStruct2, + consumedTakeOrders: [], + takeOrders: [ + { + buyToken: orderStruct2.validInputs[0].token, + buyTokenSymbol: order2.inputs[0].token.symbol, + buyTokenDecimals: orderStruct2.validInputs[0].decimals, + sellToken: orderStruct2.validOutputs[0].token, + sellTokenSymbol: order2.outputs[0].token.symbol, + sellTokenDecimals: orderStruct2.validOutputs[0].decimals, + takeOrder: { + order: orderStruct2, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + }, + ], + }, + ], + ]), + }); + const expected = new Map([]); + expected.set(`0x${"2".repeat(40)}`, ownerMap); + + const resultAsArray = Array.from(result).map((v) => [ + v[0], + Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), + ]); + const expectedAsArray = Array.from(result).map((v) => [ + v[0], + Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), + ]); + assert.deepEqual(resultAsArray, expectedAsArray); + }); }); function getOrderStruct(order) { From e05dbb0e9db553c844ec4e956019fb54d6bf8ded Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 03:09:01 +0000 Subject: [PATCH 04/32] add unit tests --- src/cli.ts | 40 +++-- src/order.ts | 107 +++++++++---- src/watcher.ts | 214 +++++++++++++------------ test/e2e/e2e.test.js | 10 +- test/orders.test.js | 364 +++++++++++++++++++++++++++++++------------ test/watcher.test.js | 323 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 813 insertions(+), 245 deletions(-) create mode 100644 test/watcher.test.js diff --git a/src/cli.ts b/src/cli.ts index 53f64222..8631613e 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,9 +14,14 @@ import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; -import { handleNewLogs, watchAllOrderbooks, WatchedOrderbookOrders } from "./watcher"; -import { getOrderbookOwnersProfileMapFromSg, prepareRoundProcessingOrders } from "./order"; +import { getOrderbookOwnersProfileMapFromSg, prepareOrdersForRound } from "./order"; import { manageAccounts, rotateProviders, sweepToMainWallet, sweepToEth } from "./account"; +import { + watchOrderbook, + watchAllOrderbooks, + WatchedOrderbookOrders, + handleOrderbooksNewLogs, +} from "./watcher"; import { diag, trace, @@ -459,7 +464,7 @@ export const main = async (argv: any, version?: string) => { orderbooksOwnersProfileMap.forEach((_, ob) => { obs.push(ob.toLowerCase()); }); - watchAllOrderbooks(obs, config.watchClient, watchedOrderbooksOrders); + const unwatchers = watchAllOrderbooks(obs, config.watchClient, watchedOrderbooksOrders); const day = 24 * 60 * 60 * 1000; let lastGasReset = Date.now() + day; @@ -476,12 +481,32 @@ export const main = async (argv: any, version?: string) => { while (true) { await tracer.startActiveSpan(`round-${counter}`, async (roundSpan) => { const roundCtx = trace.setSpan(context.active(), roundSpan); + const newMeta = await getMetaInfo(config, options.subgraph); roundSpan.setAttributes({ - ...(await getMetaInfo(config, options.subgraph)), + ...newMeta, "meta.mainAccount": config.mainAccount.account.address, "meta.gitCommitHash": process?.env?.GIT_COMMIT ?? "N/A", "meta.dockerTag": process?.env?.DOCKER_TAG ?? "N/A", }); + + // watch new obs + for (const newOb of newMeta["meta.orderbooks"]) { + const ob = newOb.toLowerCase(); + if (!obs.includes(ob)) { + obs.push(ob); + if (!watchedOrderbooksOrders[ob]) { + watchedOrderbooksOrders[ob] = { orderLogs: [] }; + } + if (!unwatchers[ob]) { + unwatchers[ob] = watchOrderbook( + ob, + config.watchClient, + watchedOrderbooksOrders[ob], + ); + } + } + } + await tracer.startActiveSpan( "check-wallet-balance", {}, @@ -533,10 +558,7 @@ export const main = async (argv: any, version?: string) => { update = true; } try { - const bundledOrders = prepareRoundProcessingOrders( - orderbooksOwnersProfileMap, - true, - ); + const bundledOrders = prepareOrdersForRound(orderbooksOwnersProfileMap, true); await rotateProviders(config, update); const roundResult = await arbRound( tracer, @@ -659,7 +681,7 @@ export const main = async (argv: any, version?: string) => { } try { // check for new orders - await handleNewLogs( + await handleOrderbooksNewLogs( orderbooksOwnersProfileMap, watchedOrderbooksOrders, config.viemClient as any as ViemClient, diff --git a/src/order.ts b/src/order.ts index 13750fe8..4b351bb7 100644 --- a/src/order.ts +++ b/src/order.ts @@ -14,7 +14,15 @@ import { OrderbooksOwnersProfileMap, } from "./types"; -export async function getPairs( +/** + * The default owner limit + */ +export const DEFAULT_OWNER_LIMIT = 25 as const; + +/** + * Get all pairs of an order + */ +export async function getOrderPairs( orderStruct: Order, viemClient: ViemClient, tokens: TokenDetails[], @@ -116,7 +124,12 @@ export async function getOrderbookOwnersProfileMapFromSg( ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { active: true, order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + takeOrders: await getOrderPairs( + orderStruct, + viemClient, + tokens, + orderDetails, + ), consumedTakeOrders: [], }); } @@ -125,11 +138,11 @@ export async function getOrderbookOwnersProfileMapFromSg( ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { active: true, order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + takeOrders: await getOrderPairs(orderStruct, viemClient, tokens, orderDetails), consumedTakeOrders: [], }); orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, orders: ordersProfileMap, }); } @@ -138,12 +151,12 @@ export async function getOrderbookOwnersProfileMapFromSg( ordersProfileMap.set(orderDetails.orderHash.toLowerCase(), { active: true, order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens, orderDetails), + takeOrders: await getOrderPairs(orderStruct, viemClient, tokens, orderDetails), consumedTakeOrders: [], }); const ownerProfileMap: OwnersProfileMap = new Map(); ownerProfileMap.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, orders: ordersProfileMap, }); orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); @@ -153,23 +166,11 @@ export async function getOrderbookOwnersProfileMapFromSg( } /** - * Get token symbol - * @param address - The address of token - * @param viemClient - The viem client + * Prepares an array of orders for a arb round by following owners limits + * @param orderbooksOwnersProfileMap - The orderbooks owners orders map + * @param shuffle - (optional) Shuffle the order of items */ -export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { - try { - return await viemClient.readContract({ - address: address as `0x${string}`, - abi: parseAbi(erc20Abi), - functionName: "symbol", - }); - } catch (error) { - return "Unknownsymbol"; - } -} - -export function prepareRoundProcessingOrders( +export function prepareOrdersForRound( orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, shuffle = true, ): BundledOrders[][] { @@ -177,27 +178,45 @@ export function prepareRoundProcessingOrders( for (const [orderbook, ownersProfileMap] of orderbooksOwnersProfileMap) { const orderbookBundledOrders: BundledOrders[] = []; for (const [, ownerProfile] of ownersProfileMap) { - let consumedLimit = ownerProfile.limit; + let remainingLimit = ownerProfile.limit; const activeOrdersProfiles = Array.from(ownerProfile.orders).filter((v) => v[1].active); const remainingOrdersPairs = activeOrdersProfiles.filter( (v) => v[1].takeOrders.length > 0, ); if (remainingOrdersPairs.length === 0) { - for (const [, orderProfile] of activeOrdersProfiles) { - orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); - } for (const [orderHash, orderProfile] of activeOrdersProfiles) { - const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); - consumedLimit -= consumingOrderPairs.length; - orderProfile.consumedTakeOrders.push(...consumingOrderPairs); - gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); + if (remainingLimit > 0) { + const consumingOrderPairs = orderProfile.takeOrders.splice( + 0, + remainingLimit, + ); + remainingLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs( + orderbook, + orderHash, + consumingOrderPairs, + orderbookBundledOrders, + ); + } } } else { for (const [orderHash, orderProfile] of remainingOrdersPairs) { - const consumingOrderPairs = orderProfile.takeOrders.splice(0, consumedLimit); - consumedLimit -= consumingOrderPairs.length; - orderProfile.consumedTakeOrders.push(...consumingOrderPairs); - gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + if (remainingLimit > 0) { + const consumingOrderPairs = orderProfile.takeOrders.splice( + 0, + remainingLimit, + ); + remainingLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs( + orderbook, + orderHash, + consumingOrderPairs, + orderbookBundledOrders, + ); + } } } } @@ -218,6 +237,9 @@ export function prepareRoundProcessingOrders( return result; } +/** + * Gathers owners orders by token pair + */ function gatherPairs( orderbook: string, orderHash: string, @@ -258,3 +280,20 @@ function gatherPairs( } } } + +/** + * Get token symbol + * @param address - The address of token + * @param viemClient - The viem client + */ +export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { + try { + return await viemClient.readContract({ + address: address as `0x${string}`, + abi: parseAbi(erc20Abi), + functionName: "symbol", + }); + } catch (error) { + return "Unknownsymbol"; + } +} diff --git a/src/watcher.ts b/src/watcher.ts index c391f30b..49e9da20 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -1,7 +1,7 @@ -import { getPairs } from "./order"; import { Span } from "@opentelemetry/api"; import { hexlify } from "ethers/lib/utils"; import { orderbookAbi as abi } from "./abis"; +import { DEFAULT_OWNER_LIMIT, getOrderPairs } from "./order"; import { parseAbi, WatchContractEventReturnType } from "viem"; import { Order, @@ -40,7 +40,13 @@ export type OrderArgsLog = { orderHash: `0x${string}`; order: Order; }; -export type WatchedOrderbookOrders = { addOrders: OrderArgsLog[]; removeOrders: OrderArgsLog[] }; +export type OrderLog = { + type: "add" | "remove"; + order: OrderArgsLog; + block: number; + logIndex: number; +}; +export type WatchedOrderbookOrders = { orderLogs: OrderLog[] }; function logToOrder(orderLog: OrderEventLog): OrderArgsLog { return { @@ -52,12 +58,12 @@ function logToOrder(orderLog: OrderEventLog): OrderArgsLog { export function toOrder(orderLog: any): Order { return { - owner: orderLog.owner, - nonce: orderLog.nonce, + owner: orderLog.owner.toLowerCase(), + nonce: orderLog.nonce.toLowerCase(), evaluable: { - interpreter: orderLog.evaluable.interpreter, - store: orderLog.evaluable.store, - bytecode: orderLog.evaluable.bytecode, + interpreter: orderLog.evaluable.interpreter.toLowerCase(), + store: orderLog.evaluable.store.toLowerCase(), + bytecode: orderLog.evaluable.bytecode.toLowerCase(), }, validInputs: orderLog.validInputs.map((v: any) => ({ token: v.token.toLowerCase(), @@ -78,6 +84,9 @@ export type UnwatchOrderbook = { unwatchRemoveOrder: WatchContractEventReturnType; }; +/** + * Applies an event watcher for a specified orderbook + */ export function watchOrderbook( orderbook: string, viemClient: ViemClient, @@ -91,9 +100,12 @@ export function watchOrderbook( onLogs: (logs) => { logs.forEach((log) => { if (log) { - watchedOrderbookOrders.addOrders.push( - logToOrder(log.args as any as OrderEventLog), - ); + watchedOrderbookOrders.orderLogs.push({ + type: "add", + logIndex: log.logIndex, + block: Number(log.blockNumber), + order: logToOrder(log.args as any as OrderEventLog), + }); } }); }, @@ -107,9 +119,12 @@ export function watchOrderbook( onLogs: (logs) => { logs.forEach((log) => { if (log) { - watchedOrderbookOrders.removeOrders.push( - logToOrder(log.args as any as OrderEventLog), - ); + watchedOrderbookOrders.orderLogs.push({ + type: "remove", + logIndex: log.logIndex, + block: Number(log.blockNumber), + order: logToOrder(log.args as any as OrderEventLog), + }); } }); }, @@ -121,29 +136,41 @@ export function watchOrderbook( }; } +/** + * Applies event watcher all known orderbooks + * @returns Unwatchers for all orderbooks + */ export function watchAllOrderbooks( orderbooks: string[], viemClient: ViemClient, watchedOrderbooksOrders: Record, ): Record { - const result: Record = {}; - for (const ob of orderbooks) { - if (!watchedOrderbooksOrders[ob]) - watchedOrderbooksOrders[ob] = { addOrders: [], removeOrders: [] }; - const res = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); - result[ob] = res; + const allUnwatchers: Record = {}; + for (const v of orderbooks) { + const ob = v.toLowerCase(); + if (!watchedOrderbooksOrders[ob]) { + watchedOrderbooksOrders[ob] = { orderLogs: [] }; + } + const unwatcher = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); + allUnwatchers[ob] = unwatcher; } - return result; + return allUnwatchers; } -export function unwatchAll(watchers: Record) { - for (const ob in watchers) { - watchers[ob].unwatchAddOrder(); - watchers[ob].unwatchRemoveOrder(); +/** + * Unwatches all orderbooks event watchers + */ +export function unwatchAllOrderbooks(unwatchers: Record) { + for (const ob in unwatchers) { + unwatchers[ob].unwatchAddOrder(); + unwatchers[ob].unwatchRemoveOrder(); } } -export async function handleNewLogs( +/** + * Hanldes all new order logs of all watched orderbooks + */ +export async function handleOrderbooksNewLogs( orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, watchedOrderbooksOrders: Record, viemClient: ViemClient, @@ -153,31 +180,30 @@ export async function handleNewLogs( ) { for (const ob in watchedOrderbooksOrders) { const watchedOrderbookLogs = watchedOrderbooksOrders[ob]; - await handleAddOrders( + const logs = watchedOrderbookLogs.orderLogs.splice(0); + // make sure logs are sorted before applying them to the map + logs.sort((a, b) => { + const block = a.block - b.block; + return block !== 0 ? block : a.logIndex - b.logIndex; + }); + await handleNewOrderLogs( ob, - watchedOrderbookLogs.addOrders.splice(0), + logs, orderbooksOwnersProfileMap, viemClient, tokens, ownerLimits, span, ); - handleRemoveOrders( - ob, - watchedOrderbookLogs.removeOrders.splice(0), - orderbooksOwnersProfileMap, - span, - ); } } /** - * Get a map of per owner orders per orderbook - * @param ordersDetails - Order details queried from subgraph + * Handles new order logs for an orderbook */ -export async function handleAddOrders( +export async function handleNewOrderLogs( orderbook: string, - addOrders: OrderArgsLog[], + orderLogs: OrderLog[], orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, viemClient: ViemClient, tokens: TokenDetails[], @@ -185,85 +211,73 @@ export async function handleAddOrders( span?: Span, ) { orderbook = orderbook.toLowerCase(); - span?.setAttribute( - "details.newOrders", - addOrders.map((v) => v.orderHash), - ); - for (let i = 0; i < addOrders.length; i++) { - const addOrderLog = addOrders[i]; - const orderStruct = addOrderLog.order; + if (orderLogs.length) { + span?.setAttribute( + `orderbooksChanges.${orderbook}.addedOrders`, + orderLogs.filter((v) => v.type === "add").map((v) => v.order.orderHash), + ); + span?.setAttribute( + `orderbooksChanges.${orderbook}.removedOrders`, + orderLogs.filter((v) => v.type === "add").map((v) => v.order.orderHash), + ); + } + for (let i = 0; i < orderLogs.length; i++) { + const orderLog = orderLogs[i].order; + const orderStruct = orderLog.order; const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); - if (orderbookOwnerProfileItem) { - const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); - if (ownerProfile) { - const order = ownerProfile.orders.get(addOrderLog.orderHash.toLowerCase()); - if (!order) { - ownerProfile.orders.set(addOrderLog.orderHash.toLowerCase(), { + if (orderLogs[i].type === "add") { + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + const order = ownerProfile.orders.get(orderLog.orderHash.toLowerCase()); + if (!order) { + ownerProfile.orders.set(orderLog.orderHash.toLowerCase(), { + active: true, + order: orderStruct, + takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), + consumedTakeOrders: [], + }); + } else { + order.active = true; + } + } else { + const ordersProfileMap: OrdersProfileMap = new Map(); + ordersProfileMap.set(orderLog.orderHash.toLowerCase(), { active: true, order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens), + takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), consumedTakeOrders: [], }); - } else { - order.active = true; + orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { + limit: + ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, + orders: ordersProfileMap, + }); } } else { const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(addOrderLog.orderHash.toLowerCase(), { + ordersProfileMap.set(orderLog.orderHash.toLowerCase(), { active: true, order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens), + takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), consumedTakeOrders: [], }); - orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, + const ownerProfileMap: OwnersProfileMap = new Map(); + ownerProfileMap.set(orderStruct.owner.toLowerCase(), { + limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, orders: ordersProfileMap, }); + orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); } } else { - const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(addOrderLog.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getPairs(orderStruct, viemClient, tokens), - consumedTakeOrders: [], - }); - const ownerProfileMap: OwnersProfileMap = new Map(); - ownerProfileMap.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? 25, - orders: ordersProfileMap, - }); - orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); - } - } -} - -/** - * Get a map of per owner orders per orderbook - * @param ordersDetails - Order details queried from subgraph - */ -export function handleRemoveOrders( - orderbook: string, - removeOrders: OrderArgsLog[], - orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, - span?: Span, -) { - orderbook = orderbook.toLowerCase(); - span?.setAttribute( - "details.removedOrders", - removeOrders.map((v) => v.orderHash), - ); - for (let i = 0; i < removeOrders.length; i++) { - const removeOrderLog = removeOrders[i]; - const orderStruct = removeOrderLog.order; - const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); - if (orderbookOwnerProfileItem) { - const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); - if (ownerProfile) { - const order = ownerProfile.orders.get(removeOrderLog.orderHash.toLowerCase()); - if (order) { - order.active = false; - order.takeOrders.push(...order.consumedTakeOrders.splice(0)); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + const order = ownerProfile.orders.get(orderLog.orderHash.toLowerCase()); + if (order) { + order.active = false; + order.takeOrders.push(...order.consumedTakeOrders.splice(0)); + } } } } diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index 515d8a77..29b8a120 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -11,14 +11,13 @@ const { trace, context } = require("@opentelemetry/api"); const { publicActions, walletActions } = require("viem"); const ERC20Artifact = require("../abis/ERC20Upgradeable.json"); const { abi: orderbookAbi } = require("../abis/OrderBook.json"); -const { prepareRoundProcessingOrders } = require("../../src/utils"); const helpers = require("@nomicfoundation/hardhat-network-helpers"); const { ProcessPairReportStatus } = require("../../src/processOrders"); const { getChainConfig, getDataFetcher } = require("../../src/config"); -const { getOrderbookOwnersProfileMapFromSg } = require("../../src/sg"); const { OTLPTraceExporter } = require("@opentelemetry/exporter-trace-otlp-http"); const { SEMRESATTRS_SERVICE_NAME } = require("@opentelemetry/semantic-conventions"); const { BasicTracerProvider, BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base"); +const { prepareOrdersForRound, getOrderbookOwnersProfileMapFromSg } = require("../../src/order"); const { arbDeploy, encodeMeta, @@ -251,7 +250,7 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; - orders = prepareRoundProcessingOrders( + orders = prepareOrdersForRound( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); @@ -585,7 +584,7 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; - orders = prepareRoundProcessingOrders( + orders = prepareOrdersForRound( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); @@ -890,7 +889,6 @@ for (let i = 0; i < testData.length; i++) { await mockServer .forPost("/rpc") .once() - // .withBodyIncluding(owners[0].address.substring(2).toLowerCase()) .thenSendJsonRpcResult( encodeQuoteResponse([ ...[tokens[1], ...t0, ...tokens.slice(2)].flatMap((v) => [ @@ -938,7 +936,7 @@ for (let i = 0; i < testData.length; i++) { config.accounts = []; config.mainAccount = bot; config.quoteRpc = [mockServer.url + "/rpc"]; - orders = prepareRoundProcessingOrders( + orders = prepareOrdersForRound( await getOrderbookOwnersProfileMapFromSg(orders, viemClient, []), false, ); diff --git a/test/orders.test.js b/test/orders.test.js index fc34e544..d22e75cc 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -1,111 +1,172 @@ const { assert } = require("chai"); const { OrderV3 } = require("../src/abis"); +const { toOrder } = require("../src/watcher"); const mockServer = require("mockttp").getLocal(); const { ethers, viem, network } = require("hardhat"); const ERC20Artifact = require("./abis/ERC20Upgradeable.json"); +const { decodeAbiParameters, parseAbiParameters } = require("viem"); const { deployOrderBookNPE2, encodeQuoteResponse } = require("./utils"); const { bundleOrders, getVaultBalance, quoteOrders, quoteSingleOrder } = require("../src/utils"); -const { toOrder } = require("../src/watcher"); -const { decodeAbiParameters, parseAbiParameters } = require("viem"); -const { getPairs, getOrderbookOwnersProfileMapFromSg } = require("../src/order"); +const { + utils: { hexlify, randomBytes, keccak256 }, +} = require("ethers"); +const { + getOrderPairs, + prepareOrdersForRound, + getOrderbookOwnersProfileMapFromSg, +} = require("../src/order"); describe("Test order details", async function () { beforeEach(() => mockServer.start(8081)); afterEach(() => mockServer.stop()); - const order1 = { - id: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", - orderHash: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", - owner: "0x0f47a0c7f86a615606ca315ad83c3e302b474bd6", - orderBytes: "", - active: true, - nonce: `0x${"0".repeat(64)}`, - orderbook: { - id: `0x${"2".repeat(40)}`, - }, - inputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", - }, + const getOrders = () => { + const order1 = { + id: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", + orderHash: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", + owner: "0x0f47a0c7f86a615606ca315ad83c3e302b474bd6", + orderBytes: "", + active: true, + nonce: `0x${"0".repeat(64)}`, + orderbook: { + id: `0x${"2".repeat(40)}`, }, - ], - outputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", + inputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, }, - }, - ], - }; - const orderStruct1 = getOrderStruct(order1); - const orderBytes1 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct1]); - order1.orderBytes = orderBytes1; - - const order2 = { - id: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", - orderHash: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", - owner: "0x0eb840e5acd0125853ad630663d3a62e673c22e6", - orderBytes: "", - active: true, - nonce: `0x${"0".repeat(64)}`, - orderbook: { - id: `0x${"2".repeat(40)}`, - }, - inputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", + ], + outputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, }, + ], + }; + const orderStruct1 = getOrderStruct(order1); + const orderBytes1 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct1]); + order1.orderBytes = orderBytes1; + + const order2 = { + id: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", + orderHash: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", + owner: "0x0eb840e5acd0125853ad630663d3a62e673c22e6", + orderBytes: "", + active: true, + nonce: `0x${"0".repeat(64)}`, + orderbook: { + id: `0x${"2".repeat(40)}`, }, - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", + inputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, }, - }, - ], - outputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, }, - }, - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", + ], + outputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, + }, + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, }, + ], + }; + const orderStruct2 = getOrderStruct(order2); + const orderBytes2 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct2]); + order2.orderBytes = orderBytes2; + + return [order1, order2]; + }; + + const getNewOrder = (orderbook, owner, token1, token2, nonce) => { + const order = { + id: "", + orderHash: "", + owner, + orderBytes: "", + active: true, + nonce: `0x${nonce.toString().repeat(64)}`, + orderbook: { + id: orderbook, }, - ], + inputs: [ + { + balance: "1", + vaultId: "0x01", + token: { + address: token1, + decimals: 6, + symbol: "NewToken1", + }, + }, + ], + outputs: [ + { + balance: "1", + vaultId: "0x01", + token: { + address: token2, + decimals: 18, + symbol: "NewToken2", + }, + }, + ], + }; + const orderStruct = getOrderStruct(order); + const orderBytes = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct]); + order.orderBytes = orderBytes; + order.struct = orderStruct; + order.id = keccak256(orderBytes); + order.orderHash = keccak256(orderBytes); + return order; }; - const orderStruct2 = getOrderStruct(order2); - const orderBytes2 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct2]); - order2.orderBytes = orderBytes2; it("should return correct order details", async function () { + const [order1, order2] = getOrders(); + const orderStruct1 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], + ); + const orderStruct2 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order2.orderBytes)[0], + ); const unbundledResult = bundleOrders([order1, order2], false, false); const unbundledExpected = [ [ @@ -123,7 +184,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes1, + order1.orderBytes, )[0], inputIOIndex: 0, outputIOIndex: 0, @@ -146,7 +207,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes2, + order2.orderBytes, )[0], inputIOIndex: 1, outputIOIndex: 0, @@ -169,7 +230,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes2, + order2.orderBytes, )[0], inputIOIndex: 0, outputIOIndex: 1, @@ -199,7 +260,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes1, + order1.orderBytes, )[0], inputIOIndex: 0, outputIOIndex: 0, @@ -211,7 +272,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes2, + order2.orderBytes, )[0], inputIOIndex: 0, outputIOIndex: 1, @@ -234,7 +295,7 @@ describe("Test order details", async function () { takeOrder: { order: ethers.utils.defaultAbiCoder.decode( [OrderV3], - orderBytes2, + order2.orderBytes, )[0], inputIOIndex: 1, outputIOIndex: 0, @@ -249,6 +310,7 @@ describe("Test order details", async function () { }); it("should get correct vault balance", async function () { + const [order1, order2] = getOrders(); const viemClient = await viem.getPublicClient(); const usdt = { address: order1.inputs[0].token.address, @@ -267,6 +329,12 @@ describe("Test order details", async function () { // deploy orderbook const orderbook = await deployOrderBookNPE2(); + const orderStruct1 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], + ); + const orderStruct2 = toOrder( + decodeAbiParameters(parseAbiParameters(OrderV3), order2.orderBytes)[0], + ); // impersonate owners and addresses with large token balances to fund the owner 1 2 // accounts with some tokens used for topping up their vaults @@ -550,18 +618,19 @@ describe("Test order details", async function () { }); it("should get order pairs", async function () { + const [order1] = getOrders(); const orderStruct = toOrder( decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], ); - const result = await getPairs(orderStruct, undefined, [], order1); + const result = await getOrderPairs(orderStruct, undefined, [], order1); const expected = [ { - buyToken: orderStruct1.validInputs[0].token, + buyToken: orderStruct.validInputs[0].token, buyTokenSymbol: order1.inputs[0].token.symbol, - buyTokenDecimals: orderStruct1.validInputs[0].decimals, - sellToken: orderStruct1.validOutputs[0].token, + buyTokenDecimals: orderStruct.validInputs[0].decimals, + sellToken: orderStruct.validOutputs[0].token, sellTokenSymbol: order1.outputs[0].token.symbol, - sellTokenDecimals: orderStruct1.validOutputs[0].decimals, + sellTokenDecimals: orderStruct.validOutputs[0].decimals, takeOrder: { order: orderStruct, inputIOIndex: 0, @@ -574,6 +643,7 @@ describe("Test order details", async function () { }); it("should make orderbook owner order map", async function () { + const [order1, order2] = getOrders(); const orderStruct1 = toOrder( decodeAbiParameters(parseAbiParameters(OrderV3), order1.orderBytes)[0], ); @@ -658,6 +728,108 @@ describe("Test order details", async function () { ]); assert.deepEqual(resultAsArray, expectedAsArray); }); + + it("should prepare orders for rounds by specified owner limits", async function () { + const orderbook = hexlify(randomBytes(20)).toLowerCase(); + const owner = hexlify(randomBytes(20)).toLowerCase(); + const token1 = hexlify(randomBytes(20)).toLowerCase(); + const token2 = hexlify(randomBytes(20)).toLowerCase(); + const [order1, order2, order3, order4, order5, order6] = [ + getNewOrder(orderbook, owner, token1, token2, 1), + getNewOrder(orderbook, owner, token1, token2, 2), + getNewOrder(orderbook, owner, token1, token2, 3), + getNewOrder(orderbook, owner, token1, token2, 4), + getNewOrder(orderbook, owner, token1, token2, 5), + getNewOrder(orderbook, owner, token1, token2, 6), + ]; + + // build orderbook owner map + const allOrders = await getOrderbookOwnersProfileMapFromSg( + [order1, order2, order3, order4, order5, order6], + undefined, + [], + { [owner]: 3 }, // set owner limit as 3 + ); + + // prepare orders for first round, ie first 3 orders should get consumed + const result1 = prepareOrdersForRound(allOrders, false); + const expected1 = [ + [ + { + buyToken: token1.toLowerCase(), + buyTokenSymbol: "NewToken1", + buyTokenDecimals: 6, + sellToken: token2.toLowerCase(), + sellTokenSymbol: "NewToken2", + sellTokenDecimals: 18, + orderbook, + takeOrders: [order1, order2, order3].map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + ], + ]; + assert.deepEqual(result1, expected1); + + // prepare orders for second round, ie second 3 orders should get consumed + const result2 = prepareOrdersForRound(allOrders, false); + const expected2 = [ + [ + { + buyToken: token1.toLowerCase(), + buyTokenSymbol: "NewToken1", + buyTokenDecimals: 6, + sellToken: token2.toLowerCase(), + sellTokenSymbol: "NewToken2", + sellTokenDecimals: 18, + orderbook, + takeOrders: [order4, order5, order6].map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + ], + ]; + assert.deepEqual(result2, expected2); + + // prepare orders for 3rd round, so should be back to consuming + // first 3 order again as 6 total order were consumed by first 2 rounds + const result3 = prepareOrdersForRound(allOrders, false); + const expected3 = [ + [ + { + buyToken: token1.toLowerCase(), + buyTokenSymbol: "NewToken1", + buyTokenDecimals: 6, + sellToken: token2.toLowerCase(), + sellTokenSymbol: "NewToken2", + sellTokenDecimals: 18, + orderbook, + takeOrders: [order1, order2, order3].map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + ], + ]; + assert.deepEqual(result3, expected3); + }); }); function getOrderStruct(order) { diff --git a/test/watcher.test.js b/test/watcher.test.js new file mode 100644 index 00000000..b4a3d400 --- /dev/null +++ b/test/watcher.test.js @@ -0,0 +1,323 @@ +const { assert } = require("chai"); +const { ethers } = require("hardhat"); +const { OrderV3 } = require("../src/abis"); +const mockServer = require("mockttp").getLocal(); +const { handleOrderbooksNewLogs } = require("../src/watcher"); +const { getOrderbookOwnersProfileMapFromSg } = require("../src/order"); +const { + utils: { hexlify, randomBytes }, +} = require("ethers"); + +describe("Test watchers", async function () { + beforeEach(() => mockServer.start(8899)); + afterEach(() => mockServer.stop()); + + const tokens = []; + function getOrderStruct(order) { + return { + nonce: order.nonce, + owner: order.owner.toLowerCase(), + evaluable: { + interpreter: `0x${"1".repeat(40)}`, + store: `0x${"2".repeat(40)}`, + bytecode: "0x1234", + }, + validInputs: order.inputs.map((v) => ({ + token: v.token.address.toLowerCase(), + decimals: v.token.decimals, + vaultId: v.vaultId, + })), + validOutputs: order.outputs.map((v) => ({ + token: v.token.address.toLowerCase(), + decimals: v.token.decimals, + vaultId: v.vaultId, + })), + }; + } + const getOrderbookOwnersProfileMap = async () => { + const order1 = { + id: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", + orderHash: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", + owner: "0x0f47a0c7f86a615606ca315ad83c3e302b474bd6", + orderBytes: "", + active: true, + nonce: `0x${"0".repeat(64)}`, + orderbook: { + id: `0x${"2".repeat(40)}`, + }, + inputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, + }, + ], + outputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, + }, + ], + }; + const orderStruct1 = getOrderStruct(order1); + const orderBytes1 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct1]); + order1.struct = orderStruct1; + order1.orderBytes = orderBytes1; + + const order2 = { + id: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", + orderHash: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", + owner: "0x0eb840e5acd0125853ad630663d3a62e673c22e6", + orderBytes: "", + active: true, + nonce: `0x${"0".repeat(64)}`, + orderbook: { + id: `0x${"2".repeat(40)}`, + }, + inputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, + }, + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, + }, + ], + outputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", + decimals: 6, + symbol: "USDT", + }, + }, + { + balance: "1", + vaultId: "1", + token: { + address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", + decimals: 18, + symbol: "WMATIC", + }, + }, + ], + }; + const orderStruct2 = getOrderStruct(order2); + const orderBytes2 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct2]); + order2.struct = orderStruct2; + order2.orderBytes = orderBytes2; + + return [ + await getOrderbookOwnersProfileMapFromSg([order1, order2], undefined, tokens, {}), + order1, + order2, + ]; + }; + + const getNewOrder = (orderbook, owner) => { + const orderHash = hexlify(randomBytes(32)); + const order = { + id: orderHash, + orderHash: orderHash, + owner, + orderBytes: "", + active: true, + nonce: `0x${"0".repeat(64)}`, + orderbook: { + id: orderbook, + }, + inputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: hexlify(randomBytes(20)), + decimals: 6, + symbol: "NewToken1", + }, + }, + ], + outputs: [ + { + balance: "1", + vaultId: "1", + token: { + address: hexlify(randomBytes(20)), + decimals: 18, + symbol: "NewToken2", + }, + }, + ], + }; + const orderStruct = getOrderStruct(order); + const orderBytes = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct]); + order.orderBytes = orderBytes; + order.struct = orderStruct; + return order; + }; + + it("should handle orderbooks new logs into orderbook owner map, add and remove", async function () { + const [orderbooksOwnersProfileMap, order1] = await getOrderbookOwnersProfileMap(); + + const newOrderbook = hexlify(randomBytes(20)); + const newOwner1 = hexlify(randomBytes(20)); + const newOrder1 = getNewOrder(newOrderbook, newOwner1); + + const newOwner2 = hexlify(randomBytes(20)); + const newOrder2 = getNewOrder(`0x${"2".repeat(40)}`, newOwner2); + + const newOrderbookLogs = { + [`0x${"2".repeat(40)}`]: { + orderLogs: [ + { + type: "remove", + block: 1, + logIndex: 1, + order: { + sender: order1.owner, + orderHash: order1.orderHash, + order: order1.struct, + }, + }, + { + type: "add", + block: 2, + logIndex: 1, + order: { + sender: newOwner2, + orderHash: newOrder2.orderHash, + order: newOrder2.struct, + }, + }, + ], + }, + [newOrderbook]: { + orderLogs: [ + { + type: "add", + block: 2, + logIndex: 1, + order: { + sender: newOwner1, + orderHash: newOrder1.orderHash, + order: newOrder1.struct, + }, + }, + ], + }, + }; + await handleOrderbooksNewLogs( + orderbooksOwnersProfileMap, + newOrderbookLogs, + undefined, + tokens, + {}, + ); + + const expectedMap = (await getOrderbookOwnersProfileMap())[0]; + expectedMap + .get(`0x${"2".repeat(40)}`) + .get(order1.owner.toLowerCase()) + .orders.get(order1.orderHash.toLowerCase()).active = false; + expectedMap.get(`0x${"2".repeat(40)}`).set(newOwner2, { + limits: 25, + orders: new Map([ + [ + newOrder2.orderHash.toLowerCase(), + { + active: true, + order: newOrder2, + consumedTakeOrders: [], + takeOrders: [ + { + buyToken: newOrder2.struct.validInputs[0].token, + buyTokenSymbol: newOrder2.inputs[0].token.symbol, + buyTokenDecimals: newOrder2.struct.validInputs[0].decimals, + sellToken: newOrder2.struct.validOutputs[0].token, + sellTokenSymbol: newOrder2.outputs[0].token.symbol, + sellTokenDecimals: newOrder2.struct.validOutputs[0].decimals, + takeOrder: { + order: newOrder2.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + }, + ], + }, + ], + ]), + }); + expectedMap.set( + newOrderbook, + new Map([ + [ + newOwner1, + { + limits: 25, + orders: new Map([ + [ + newOrder1.orderHash.toLowerCase(), + { + active: true, + order: newOrder1, + consumedTakeOrders: [], + takeOrders: [ + { + buyToken: newOrder1.struct.validInputs[0].token, + buyTokenSymbol: newOrder1.inputs[0].token.symbol, + buyTokenDecimals: + newOrder1.struct.validInputs[0].decimals, + sellToken: newOrder1.struct.validOutputs[0].token, + sellTokenSymbol: newOrder1.outputs[0].token.symbol, + sellTokenDecimals: + newOrder1.struct.validOutputs[0].decimals, + takeOrder: { + order: newOrder1.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + }, + ], + }, + ], + ]), + }, + ], + ]), + ); + + const result = Array.from(orderbooksOwnersProfileMap).map((v) => [ + v[0], + Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), + ]); + const expected = Array.from(expectedMap).map((v) => [ + v[0], + Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), + ]); + assert.deepEqual(result, expected); + }); +}); From ef684c4f6497dd4caa5dbf42b902ddfa744e8309 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 04:54:37 +0000 Subject: [PATCH 05/32] update --- src/cli.ts | 21 +++++++++++++++++---- src/utils.ts | 13 +++++++++++++ src/watcher.ts | 7 +++++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 8631613e..598b08c9 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -4,12 +4,12 @@ import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; -import { sleep, getOrdersTokens } from "./utils"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; +import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; @@ -214,10 +214,23 @@ const getOptions = async (argv: any, version?: string) => { const profiles: Record = {}; cmdOptions.ownerProfile.forEach((v: string) => { const parsed = v.split("="); - if (parsed.length !== 2) throw "Invalid owner profile"; - if (!/^[0-9]+$/.test(parsed[1])) + if (parsed.length !== 2) { + throw "Invalid owner profile, must be in form of 'ownerAddress=limitValue'"; + } + if (!ethers.utils.isAddress(parsed[0])) { + throw `Invalid owner address: ${parsed[0]}`; + } + if (!isBigNumberish(parsed[1]) && parsed[1] !== "max") { throw "Invalid owner profile limit, must be an integer gte 0"; - profiles[parsed[0].toLowerCase()] = Number(parsed[1]); + } + if (parsed[1] === "max") { + profiles[parsed[0].toLowerCase()] = Number.MAX_SAFE_INTEGER; + } else { + const limit = BigNumber.from(parsed[1]); + profiles[parsed[0].toLowerCase()] = limit.gte(Number.MAX_SAFE_INTEGER.toString()) + ? Number.MAX_SAFE_INTEGER + : limit.toNumber(); + } }); cmdOptions.ownerProfile = profiles; } diff --git a/src/utils.ts b/src/utils.ts index dcea6297..898061c0 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -10,6 +10,7 @@ import { parseAbi, PublicClient, TransactionReceipt } from "viem"; import { doQuoteTargets, QuoteTarget } from "@rainlanguage/orderbook/quote"; import { BotConfig, BundledOrders, OwnedOrder, TakeOrder, TokenDetails } from "./types"; import { DataFetcher, DataFetcherOptions, LiquidityProviders, Router } from "sushi/router"; +import { isBytes, isHexString } from "ethers/lib/utils"; export function RPoolFilter(pool: any) { return !BlackList.includes(pool.address) && !BlackList.includes(pool.address.toLowerCase()); @@ -1280,3 +1281,15 @@ export function getMarketQuote( }; } } + +export function isBigNumberish(value: any): value is BigNumberish { + return ( + value != null && + (BigNumber.isBigNumber(value) || + (typeof value === "number" && value % 1 === 0) || + (typeof value === "string" && !!value.match(/^-?[0-9]+$/)) || + isHexString(value) || + typeof value === "bigint" || + isBytes(value)) + ); +} diff --git a/src/watcher.ts b/src/watcher.ts index 49e9da20..c09615a4 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -45,13 +45,14 @@ export type OrderLog = { order: OrderArgsLog; block: number; logIndex: number; + txHash: string; }; export type WatchedOrderbookOrders = { orderLogs: OrderLog[] }; function logToOrder(orderLog: OrderEventLog): OrderArgsLog { return { - sender: orderLog.sender, - orderHash: orderLog.orderHash, + sender: orderLog.sender.toLowerCase() as `0x${string}`, + orderHash: orderLog.orderHash.toLowerCase() as `0x${string}`, order: toOrder(orderLog.order), }; } @@ -104,6 +105,7 @@ export function watchOrderbook( type: "add", logIndex: log.logIndex, block: Number(log.blockNumber), + txHash: log.transactionHash, order: logToOrder(log.args as any as OrderEventLog), }); } @@ -123,6 +125,7 @@ export function watchOrderbook( type: "remove", logIndex: log.logIndex, block: Number(log.blockNumber), + txHash: log.transactionHash, order: logToOrder(log.args as any as OrderEventLog), }); } From 8d90b7a8003e666e691d8d0437d6ca340974b962 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 05:17:15 +0000 Subject: [PATCH 06/32] Update processOrders.ts --- src/processOrders.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index 27aae63d..5966ff82 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -21,7 +21,6 @@ import { ProcessPairResult, } from "./types"; import { - sleep, toNumber, getIncome, getEthPrice, @@ -331,7 +330,6 @@ export const processOrders = async ( // rotate the accounts once they are used once rotateAccounts(accounts); - await sleep(2000); } } } From a11dbb2d8fd375172ddabf2a52e89b22de2e39b3 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 05:51:45 +0000 Subject: [PATCH 07/32] update --- src/abis.ts | 2 +- src/watcher.ts | 50 +++++++++----------------------------------------- 2 files changed, 10 insertions(+), 42 deletions(-) diff --git a/src/abis.ts b/src/abis.ts index 5a12efc0..9877aa93 100644 --- a/src/abis.ts +++ b/src/abis.ts @@ -33,8 +33,8 @@ export const ClearConfig = */ export const orderbookAbi = [ `event AddOrderV2(address sender, bytes32 orderHash, ${OrderV3} order)`, - `event AfterClear(address sender, ${ClearStateChange} clearStateChange)`, `event RemoveOrderV2(address sender, bytes32 orderHash, ${OrderV3} order)`, + `event AfterClear(address sender, ${ClearStateChange} clearStateChange)`, "function vaultBalance(address owner, address token, uint256 vaultId) external view returns (uint256 balance)", `function deposit2(address token, uint256 vaultId, uint256 amount, ${TaskV1}[] calldata tasks) external`, `function addOrder2(${OrderConfigV3} calldata config, ${TaskV1}[] calldata tasks) external returns (bool stateChanged)`, diff --git a/src/watcher.ts b/src/watcher.ts index c09615a4..4dfee502 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -79,11 +79,7 @@ export function toOrder(orderLog: any): Order { }; } -export const orderbookAbi = parseAbi(abi); -export type UnwatchOrderbook = { - unwatchAddOrder: WatchContractEventReturnType; - unwatchRemoveOrder: WatchContractEventReturnType; -}; +export const orderbookAbi = parseAbi([abi[0], abi[1]]); /** * Applies an event watcher for a specified orderbook @@ -92,17 +88,16 @@ export function watchOrderbook( orderbook: string, viemClient: ViemClient, watchedOrderbookOrders: WatchedOrderbookOrders, -): UnwatchOrderbook { - const unwatchAddOrder = viemClient.watchContractEvent({ +): WatchContractEventReturnType { + return viemClient.watchContractEvent({ address: orderbook as `0x${string}`, abi: orderbookAbi, - eventName: "AddOrderV2", pollingInterval: 30_000, onLogs: (logs) => { logs.forEach((log) => { if (log) { watchedOrderbookOrders.orderLogs.push({ - type: "add", + type: log.eventName === "AddOrderV2" ? "add" : "remove", logIndex: log.logIndex, block: Number(log.blockNumber), txHash: log.transactionHash, @@ -112,31 +107,6 @@ export function watchOrderbook( }); }, }); - - const unwatchRemoveOrder = viemClient.watchContractEvent({ - address: orderbook as `0x${string}`, - abi: orderbookAbi, - eventName: "RemoveOrderV2", - pollingInterval: 30_000, - onLogs: (logs) => { - logs.forEach((log) => { - if (log) { - watchedOrderbookOrders.orderLogs.push({ - type: "remove", - logIndex: log.logIndex, - block: Number(log.blockNumber), - txHash: log.transactionHash, - order: logToOrder(log.args as any as OrderEventLog), - }); - } - }); - }, - }); - - return { - unwatchAddOrder, - unwatchRemoveOrder, - }; } /** @@ -147,15 +117,14 @@ export function watchAllOrderbooks( orderbooks: string[], viemClient: ViemClient, watchedOrderbooksOrders: Record, -): Record { - const allUnwatchers: Record = {}; +): Record { + const allUnwatchers: Record = {}; for (const v of orderbooks) { const ob = v.toLowerCase(); if (!watchedOrderbooksOrders[ob]) { watchedOrderbooksOrders[ob] = { orderLogs: [] }; } - const unwatcher = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); - allUnwatchers[ob] = unwatcher; + allUnwatchers[ob] = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); } return allUnwatchers; } @@ -163,10 +132,9 @@ export function watchAllOrderbooks( /** * Unwatches all orderbooks event watchers */ -export function unwatchAllOrderbooks(unwatchers: Record) { +export function unwatchAllOrderbooks(unwatchers: Record) { for (const ob in unwatchers) { - unwatchers[ob].unwatchAddOrder(); - unwatchers[ob].unwatchRemoveOrder(); + unwatchers[ob]?.(); } } From 72c5a35d6cecabf597633ca9cc30b1a139f9e6af Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 06:34:37 +0000 Subject: [PATCH 08/32] Update cli.ts --- src/cli.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli.ts b/src/cli.ts index 598b08c9..76370e84 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -573,6 +573,7 @@ export const main = async (argv: any, version?: string) => { try { const bundledOrders = prepareOrdersForRound(orderbooksOwnersProfileMap, true); await rotateProviders(config, update); + roundSpan.setAttribute("details.rpc", config.rpc); const roundResult = await arbRound( tracer, roundCtx, From 0998aad718dcfe254eb115627c7f2a003c3e6a9d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 26 Oct 2024 06:47:48 +0000 Subject: [PATCH 09/32] Update order.ts --- src/order.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/order.ts b/src/order.ts index 4b351bb7..628330fa 100644 --- a/src/order.ts +++ b/src/order.ts @@ -1,6 +1,6 @@ import { SgOrder } from "./query"; import { toOrder } from "./watcher"; -import { shuffleArray } from "./utils"; +import { shuffleArray, sleep } from "./utils"; import { erc20Abi, OrderV3 } from "./abis"; import { decodeAbiParameters, parseAbi, parseAbiParameters } from "viem"; import { @@ -287,13 +287,17 @@ function gatherPairs( * @param viemClient - The viem client */ export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { - try { - return await viemClient.readContract({ - address: address as `0x${string}`, - abi: parseAbi(erc20Abi), - functionName: "symbol", - }); - } catch (error) { - return "Unknownsymbol"; + // 3 retries + for (let i = 0; i < 3; i++) { + try { + return await viemClient.readContract({ + address: address as `0x${string}`, + abi: parseAbi(erc20Abi), + functionName: "symbol", + }); + } catch { + await sleep(10_000); + } } + return "UnknownSymbol"; } From f19d5086ae79d0e1363300521c820cb826b04368 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sun, 27 Oct 2024 04:47:56 +0000 Subject: [PATCH 10/32] update sushi version --- lib/sushiswap | 2 +- package-lock.json | 170 ++++++++++++++++++++++++++++++++++++++-------- package.json | 2 +- src/cli.ts | 64 ++++++++++------- src/utils.ts | 19 ++++-- src/watcher.ts | 2 + test/cli.test.js | 2 +- 7 files changed, 202 insertions(+), 59 deletions(-) diff --git a/lib/sushiswap b/lib/sushiswap index 0b6e4452..97c0a77d 160000 --- a/lib/sushiswap +++ b/lib/sushiswap @@ -1 +1 @@ -Subproject commit 0b6e44528b80de8660e07f0619d68d19c034ee13 +Subproject commit 97c0a77d8fd5137690531afb29c10e25cb27af8a diff --git a/package-lock.json b/package-lock.json index 6e383048..42d9673a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "dotenv": "^16.0.3", "ethers": "5.7.0", "sushi": "./lib/sushiswap/packages/sushi", - "viem": "=2.8.14" + "viem": "=2.21.35" }, "bin": { "arb-bot": "arb-bot.js" @@ -93,9 +93,9 @@ "@wagmi/core": "2.6.17", "ts-node": "10.9.2", "typescript": "5.2.2", - "viem": "2.8.14", + "viem": "2.21.35", "vitest": "0.34.6", - "zod": "3.21.4" + "zod": "3.22.0" }, "peerDependencies": { "viem": "*", @@ -124,9 +124,9 @@ } }, "lib/sushiswap/packages/sushi/node_modules/zod": { - "version": "3.21.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", - "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", + "version": "3.22.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.0.tgz", + "integrity": "sha512-y5KZY/ssf5n7hCGDGGtcJO/EBJEm5Pa+QQvFBeyMOtnFYOSflalxIFFvdaYevPhePcmcKC4aTbFkCcXN7D0O8Q==", "dev": true, "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -135,7 +135,8 @@ "node_modules/@adraffy/ens-normalize": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.10.0.tgz", - "integrity": "sha512-nA9XHtlAkYfJxY7bce8DcN7eKxWWCWkU+1GR9d+U6MbNpfwQp8TI7vqOsBsMcHoT4mBu2kypKoSKnghEzOOq5Q==" + "integrity": "sha512-nA9XHtlAkYfJxY7bce8DcN7eKxWWCWkU+1GR9d+U6MbNpfwQp8TI7vqOsBsMcHoT4mBu2kypKoSKnghEzOOq5Q==", + "dev": true }, "node_modules/@babel/parser": { "version": "7.24.5", @@ -2258,6 +2259,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.2.0.tgz", "integrity": "sha512-oYclrNgRaM9SsBUBVbb8M6DTV7ZHRTKugureoYEncY5c65HOmRzvSiTE3y5CYaPYJA/GVkrhXEoF0M3Ya9PMnw==", + "dev": true, "dependencies": { "@noble/hashes": "1.3.2" }, @@ -2269,6 +2271,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.3.2.tgz", "integrity": "sha512-MVC8EAQp7MvEcm30KWENFjgR+Mkmf+D189XJTkFIlwohU5hcBbn1ZkKq7KVTi2Hme3PMGF390DaL52beVrIihQ==", + "dev": true, "engines": { "node": ">= 16" }, @@ -3765,9 +3768,9 @@ ] }, "node_modules/@scure/base": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.6.tgz", - "integrity": "sha512-ok9AWwhcgYuGG3Zfhyqg+zwl+Wn5uE+dwC0NV/2qQkx4dABbb/bx96vWu8NSj+BNjjSjno+JRYRjle1jV08k3g==", + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.1.9.tgz", + "integrity": "sha512-8YKhl8GHiNI/pU2VMaofa2Tor7PJRAjwQLBBuilkJ9L5+13yVbC7JO/wS7piioAvPSwR3JKM1IJ/u4xQzbcXKg==", "funding": { "url": "https://paulmillr.com/funding/" } @@ -3776,6 +3779,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.3.2.tgz", "integrity": "sha512-N1ZhksgwD3OBlwTv3R6KFEcPojl/W4ElJOeCZdi+vuI5QmTFwLq3OFf2zd2ROpKvxFdgZ6hUpb0dx9bVNEwYCA==", + "dev": true, "dependencies": { "@noble/curves": "~1.2.0", "@noble/hashes": "~1.3.2", @@ -3789,6 +3793,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.2.1.tgz", "integrity": "sha512-Z3/Fsz1yr904dduJD0NpiyRHhRYHdcnyh73FZWiV+/qhWi83wNJ3NWolYqCEN+ZWsUz2TWwajJggcRE9r1zUYg==", + "dev": true, "dependencies": { "@noble/hashes": "~1.3.0", "@scure/base": "~1.1.0" @@ -4781,9 +4786,9 @@ "peer": true }, "node_modules/abitype": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.0.0.tgz", - "integrity": "sha512-NMeMah//6bJ56H5XRj8QCV4AwuW6hB6zqz2LnhhLdcWVQOsXki6/Pn3APeqxCma62nXIcmZWdu1DlHWS74umVQ==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.0.6.tgz", + "integrity": "sha512-MMSqYh4+C/aVqI2RQaWqbvI4Kxo5cQV40WQ4QFtDnNzCkqChm8MuENhElmynZlO0qUy/ObkEUaXtKqYnx1Kp3A==", "funding": { "url": "https://github.com/sponsors/wevm" }, @@ -19664,6 +19669,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.3.tgz", "integrity": "sha512-2cKei4vlmg2cxEjm3wVSqn8pcoRF/LX/wpifuuNquFO4SQmPwarClT+SUCA2lt+l581tTeZIPIZuIDo2jWN1fg==", + "dev": true, "funding": [ { "type": "github", @@ -24874,9 +24880,9 @@ } }, "node_modules/viem": { - "version": "2.8.14", - "resolved": "https://registry.npmjs.org/viem/-/viem-2.8.14.tgz", - "integrity": "sha512-K5u9OoyPQ7W8VPa6xY2m7oazuhemp0xuK9Ur8AkaXHtcusism9keTXDDaCw6WWFK3YR9HSojHJOtuVQqvRz0ug==", + "version": "2.21.35", + "resolved": "https://registry.npmjs.org/viem/-/viem-2.21.35.tgz", + "integrity": "sha512-f3EFc5JILeA9veuNymUN8HG/nKP9ykC0NCgwFrZWuxcCc822GaP0IEnkRBsHGqmjwbz//FxJFmvtx7TBcdVs0A==", "funding": [ { "type": "github", @@ -24884,14 +24890,15 @@ } ], "dependencies": { - "@adraffy/ens-normalize": "1.10.0", - "@noble/curves": "1.2.0", - "@noble/hashes": "1.3.2", - "@scure/bip32": "1.3.2", - "@scure/bip39": "1.2.1", - "abitype": "1.0.0", - "isows": "1.0.3", - "ws": "8.13.0" + "@adraffy/ens-normalize": "1.11.0", + "@noble/curves": "1.6.0", + "@noble/hashes": "1.5.0", + "@scure/bip32": "1.5.0", + "@scure/bip39": "1.4.0", + "abitype": "1.0.6", + "isows": "1.0.6", + "webauthn-p256": "0.0.10", + "ws": "8.18.0" }, "peerDependencies": { "typescript": ">=5.0.4" @@ -24902,10 +24909,79 @@ } } }, + "node_modules/viem/node_modules/@adraffy/ens-normalize": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.0.tgz", + "integrity": "sha512-/3DDPKHqqIqxUULp8yP4zODUY1i+2xvVWsv8A79xGWdCAG+8sb0hRh0Rk2QyOJUnnbyPUAZYcpBuRe3nS2OIUg==" + }, + "node_modules/viem/node_modules/@noble/curves": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.6.0.tgz", + "integrity": "sha512-TlaHRXDehJuRNR9TfZDNQ45mMEd5dwUwmicsafcIX4SsNiqnCHKjE/1alYPd/lDRVhxdhUAlv8uEhMCI5zjIJQ==", + "dependencies": { + "@noble/hashes": "1.5.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/viem/node_modules/@noble/hashes": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.5.0.tgz", + "integrity": "sha512-1j6kQFb7QRru7eKN3ZDvRcP13rugwdxZqCjbiAVZfIJwgj2A65UmT4TgARXGlXgnRkORLTDTrO19ZErt7+QXgA==", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/viem/node_modules/@scure/bip32": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.5.0.tgz", + "integrity": "sha512-8EnFYkqEQdnkuGBVpCzKxyIwDCBLDVj3oiX0EKUFre/tOjL/Hqba1D6n/8RcmaQy4f95qQFrO2A8Sr6ybh4NRw==", + "dependencies": { + "@noble/curves": "~1.6.0", + "@noble/hashes": "~1.5.0", + "@scure/base": "~1.1.7" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/viem/node_modules/@scure/bip39": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.4.0.tgz", + "integrity": "sha512-BEEm6p8IueV/ZTfQLp/0vhw4NPnT9oWf5+28nvmeUICjP99f4vr2d+qc7AVGDDtwRep6ifR43Yed9ERVmiITzw==", + "dependencies": { + "@noble/hashes": "~1.5.0", + "@scure/base": "~1.1.8" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/viem/node_modules/isows": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.6.tgz", + "integrity": "sha512-lPHCayd40oW98/I0uvgaHKWCSvkzY27LjWLbtzOm64yQ+G3Q5npjjbdppU65iZXkK1Zt+kH9pfegli0AYfwYYw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "peerDependencies": { + "ws": "*" + } + }, "node_modules/viem/node_modules/ws": { - "version": "8.13.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", - "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, @@ -25174,6 +25250,46 @@ "@scure/bip39": "1.2.2" } }, + "node_modules/webauthn-p256": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/webauthn-p256/-/webauthn-p256-0.0.10.tgz", + "integrity": "sha512-EeYD+gmIT80YkSIDb2iWq0lq2zbHo1CxHlQTeJ+KkCILWpVy3zASH3ByD4bopzfk0uCwXxLqKGLqp2W4O28VFA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "dependencies": { + "@noble/curves": "^1.4.0", + "@noble/hashes": "^1.4.0" + } + }, + "node_modules/webauthn-p256/node_modules/@noble/curves": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.6.0.tgz", + "integrity": "sha512-TlaHRXDehJuRNR9TfZDNQ45mMEd5dwUwmicsafcIX4SsNiqnCHKjE/1alYPd/lDRVhxdhUAlv8uEhMCI5zjIJQ==", + "dependencies": { + "@noble/hashes": "1.5.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/webauthn-p256/node_modules/@noble/hashes": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.5.0.tgz", + "integrity": "sha512-1j6kQFb7QRru7eKN3ZDvRcP13rugwdxZqCjbiAVZfIJwgj2A65UmT4TgARXGlXgnRkORLTDTrO19ZErt7+QXgA==", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 22566480..8f37bd6f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "dotenv": "^16.0.3", "ethers": "5.7.0", "sushi": "./lib/sushiswap/packages/sushi", - "viem": "=2.8.14" + "viem": "=2.21.35" }, "devDependencies": { "@nomicfoundation/hardhat-network-helpers": "^1.0.8", diff --git a/src/cli.ts b/src/cli.ts index 76370e84..ef73014d 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,11 +9,11 @@ import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; -import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { sleep, getOrdersTokens, isBigNumberish, getblockNumber } from "./utils"; import { getOrderbookOwnersProfileMapFromSg, prepareOrdersForRound } from "./order"; import { manageAccounts, rotateProviders, sweepToMainWallet, sweepToEth } from "./account"; import { @@ -368,7 +368,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? } const poolUpdateInterval = _poolUpdateInterval * 60 * 1000; let ordersDetails: SgOrder[] = []; - if (!process?.env?.TEST) + if (!process?.env?.CLI_STARTUP_TEST) { for (let i = 0; i < 20; i++) { try { ordersDetails = await getOrderDetails(options.subgraph, { @@ -382,6 +382,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? else throw e; } } + } const tokens = getOrdersTokens(ordersDetails); options.tokens = [...tokens]; @@ -394,6 +395,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? tracer, ctx, ); + const blockNumber = (await getblockNumber(config.viemClient as any as ViemClient)) ?? 0n; return { roundGap, @@ -407,6 +409,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? (options as CliOptions).ownerProfile, ), tokens, + blockNumber, }; } @@ -445,33 +448,41 @@ export const main = async (argv: any, version?: string) => { const tracer = provider.getTracer("arb-bot-tracer"); // parse cli args and startup bot configuration - const { roundGap, options, poolUpdateInterval, config, orderbooksOwnersProfileMap, tokens } = - await tracer.startActiveSpan("startup", async (startupSpan) => { - const ctx = trace.setSpan(context.active(), startupSpan); - try { - const result = await startup(argv, version, tracer, ctx); - startupSpan.setStatus({ code: SpanStatusCode.OK }); - startupSpan.end(); - return result; - } catch (e: any) { - const snapshot = errorSnapshot("", e); - startupSpan.setAttribute("severity", ErrorSeverity.HIGH); - startupSpan.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); - startupSpan.recordException(e); + const { + roundGap, + options, + poolUpdateInterval, + config, + orderbooksOwnersProfileMap, + tokens, + blockNumber: bn, + } = await tracer.startActiveSpan("startup", async (startupSpan) => { + const ctx = trace.setSpan(context.active(), startupSpan); + try { + const result = await startup(argv, version, tracer, ctx); + startupSpan.setStatus({ code: SpanStatusCode.OK }); + startupSpan.end(); + return result; + } catch (e: any) { + const snapshot = errorSnapshot("", e); + startupSpan.setAttribute("severity", ErrorSeverity.HIGH); + startupSpan.setStatus({ code: SpanStatusCode.ERROR, message: snapshot }); + startupSpan.recordException(e); - // end this span and wait for it to finish - startupSpan.end(); - await sleep(20000); + // end this span and wait for it to finish + startupSpan.end(); + await sleep(20000); - // flush and close the otel connection. - await exporter.shutdown(); - await sleep(10000); + // flush and close the otel connection. + await exporter.shutdown(); + await sleep(10000); - // reject the promise that makes the cli process to exit with error - return Promise.reject(e); - } - }); + // reject the promise that makes the cli process to exit with error + return Promise.reject(e); + } + }); + let blockNumber = bn; const obs: string[] = []; const watchedOrderbooksOrders: Record = {}; orderbooksOwnersProfileMap.forEach((_, ob) => { @@ -515,10 +526,13 @@ export const main = async (argv: any, version?: string) => { ob, config.watchClient, watchedOrderbooksOrders[ob], + blockNumber, ); } } } + const tempBn = await getblockNumber(config.viemClient as any as ViemClient); + if (tempBn !== undefined) blockNumber = (tempBn * 95n) / 100n; await tracer.startActiveSpan( "check-wallet-balance", diff --git a/src/utils.ts b/src/utils.ts index 898061c0..ad6cd4da 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -4,13 +4,13 @@ import { RouteLeg } from "sushi/tines"; import { getDataFetcher } from "./config"; import { Token, Type } from "sushi/currency"; import BlackList from "./pool-blacklist.json"; +import { isBytes, isHexString } from "ethers/lib/utils"; import { BigNumber, BigNumberish, ethers } from "ethers"; import { erc20Abi, orderbookAbi, OrderV3 } from "./abis"; import { parseAbi, PublicClient, TransactionReceipt } from "viem"; import { doQuoteTargets, QuoteTarget } from "@rainlanguage/orderbook/quote"; -import { BotConfig, BundledOrders, OwnedOrder, TakeOrder, TokenDetails } from "./types"; +import { BotConfig, BundledOrders, OwnedOrder, TakeOrder, TokenDetails, ViemClient } from "./types"; import { DataFetcher, DataFetcherOptions, LiquidityProviders, Router } from "sushi/router"; -import { isBytes, isHexString } from "ethers/lib/utils"; export function RPoolFilter(pool: any) { return !BlackList.includes(pool.address) && !BlackList.includes(pool.address.toLowerCase()); @@ -708,7 +708,7 @@ export async function getVaultBalance( address: orderbookAddress as `0x${string}`, allowFailure: false, chainId: viemClient.chain!.id, - abi: parseAbi(orderbookAbi), + abi: parseAbi([orderbookAbi[3]]), functionName: "vaultBalance", args: [ // owner @@ -1210,7 +1210,7 @@ export async function checkOwnedOrders( address: v.orderbook, allowFailure: false, chainId: config.chain.id, - abi: parseAbi(orderbookAbi), + abi: parseAbi([orderbookAbi[3]]), functionName: "vaultBalance", args: [ // owner @@ -1293,3 +1293,14 @@ export function isBigNumberish(value: any): value is BigNumberish { isBytes(value)) ); } + +export async function getblockNumber(viemClient: ViemClient): Promise { + for (let i = 0; i < 3; i++) { + try { + return await viemClient.getBlockNumber(); + } catch (e) { + await sleep(5_000); + } + } + return; +} diff --git a/src/watcher.ts b/src/watcher.ts index 4dfee502..dfa61916 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -88,11 +88,13 @@ export function watchOrderbook( orderbook: string, viemClient: ViemClient, watchedOrderbookOrders: WatchedOrderbookOrders, + fromBlock?: bigint, ): WatchContractEventReturnType { return viemClient.watchContractEvent({ address: orderbook as `0x${string}`, abi: orderbookAbi, pollingInterval: 30_000, + fromBlock, onLogs: (logs) => { logs.forEach((log) => { if (log) { diff --git a/test/cli.test.js b/test/cli.test.js index 47302b9c..2c2117f4 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -54,7 +54,7 @@ describe("Test cli", async function () { }); it("test cli startup", async function () { - process.env.TEST = true; + process.env.CLI_STARTUP_TEST = true; try { await startup(["", ""]); assert.fail("expected to fail, but resolved"); From c28eeffda06a2cabc39525357fa38da65e42fa33 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 28 Oct 2024 12:38:51 +0000 Subject: [PATCH 11/32] Update sushiswap --- lib/sushiswap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sushiswap b/lib/sushiswap index 97c0a77d..72289f46 160000 --- a/lib/sushiswap +++ b/lib/sushiswap @@ -1 +1 @@ -Subproject commit 97c0a77d8fd5137690531afb29c10e25cb27af8a +Subproject commit 72289f46a36ee3779a215e909518214897a99912 From 090985b8bfa0f37fff5caed50baa43b075d7ca8e Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 28 Oct 2024 13:52:27 +0000 Subject: [PATCH 12/32] update --- package-lock.json | 5 +--- src/cli.ts | 60 ++++++++++++++++++++++++++++------------------- src/watcher.ts | 35 +++++++++++++++++++++------ 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/package-lock.json b/package-lock.json index 42d9673a..221431f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14263,7 +14263,6 @@ "hasInstallScript": true, "inBundle": true, "license": "MIT", - "peer": true, "dependencies": { "node-addon-api": "^2.0.0", "node-gyp-build": "^4.2.0" @@ -14893,8 +14892,7 @@ "version": "2.0.2", "dev": true, "inBundle": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/ganache-core/node_modules/node-fetch": { "version": "2.1.2", @@ -14910,7 +14908,6 @@ "dev": true, "inBundle": true, "license": "MIT", - "peer": true, "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", diff --git a/src/cli.ts b/src/cli.ts index ef73014d..88d2c948 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -186,30 +186,33 @@ const getOptions = async (argv: any, version?: string) => { .opts(); // assigning specified options from cli/env - cmdOptions.key = cmdOptions.key || ENV_OPTIONS.key; - cmdOptions.mnemonic = cmdOptions.mnemonic || ENV_OPTIONS.mnemonic; - cmdOptions.rpc = cmdOptions.rpc || ENV_OPTIONS.rpc; - cmdOptions.arbAddress = cmdOptions.arbAddress || ENV_OPTIONS.arbAddress; - cmdOptions.genericArbAddress = cmdOptions.genericArbAddress || ENV_OPTIONS.genericArbAddress; - cmdOptions.orderbookAddress = cmdOptions.orderbookAddress || ENV_OPTIONS.orderbookAddress; - cmdOptions.subgraph = cmdOptions.subgraph || ENV_OPTIONS.subgraph; - cmdOptions.lps = cmdOptions.lps || ENV_OPTIONS.lps; - cmdOptions.gasCoverage = cmdOptions.gasCoverage || ENV_OPTIONS.gasCoverage; - cmdOptions.orderHash = cmdOptions.orderHash || ENV_OPTIONS.orderHash; - cmdOptions.orderOwner = cmdOptions.orderOwner || ENV_OPTIONS.orderOwner; - cmdOptions.sleep = cmdOptions.sleep || ENV_OPTIONS.sleep; - cmdOptions.maxRatio = cmdOptions.maxRatio || ENV_OPTIONS.maxRatio; - cmdOptions.flashbotRpc = cmdOptions.flashbotRpc || ENV_OPTIONS.flashbotRpc; - cmdOptions.timeout = cmdOptions.timeout || ENV_OPTIONS.timeout; - cmdOptions.hops = cmdOptions.hops || ENV_OPTIONS.hops; - cmdOptions.retries = cmdOptions.retries || ENV_OPTIONS.retries; - cmdOptions.poolUpdateInterval = cmdOptions.poolUpdateInterval || ENV_OPTIONS.poolUpdateInterval; - cmdOptions.walletCount = cmdOptions.walletCount || ENV_OPTIONS.walletCount; - cmdOptions.topupAmount = cmdOptions.topupAmount || ENV_OPTIONS.topupAmount; - cmdOptions.selfFundOrders = cmdOptions.selfFundOrders || ENV_OPTIONS.selfFundOrders; - cmdOptions.botMinBalance = cmdOptions.botMinBalance || ENV_OPTIONS.botMinBalance; - cmdOptions.ownerProfile = cmdOptions.ownerProfile || ENV_OPTIONS.ownerProfile; - cmdOptions.bundle = cmdOptions.bundle ? ENV_OPTIONS.bundle : false; + cmdOptions.key = cmdOptions.key || getEnv(ENV_OPTIONS.key); + cmdOptions.mnemonic = cmdOptions.mnemonic || getEnv(ENV_OPTIONS.mnemonic); + cmdOptions.rpc = cmdOptions.rpc || getEnv(ENV_OPTIONS.rpc); + cmdOptions.arbAddress = cmdOptions.arbAddress || getEnv(ENV_OPTIONS.arbAddress); + cmdOptions.genericArbAddress = + cmdOptions.genericArbAddress || getEnv(ENV_OPTIONS.genericArbAddress); + cmdOptions.orderbookAddress = + cmdOptions.orderbookAddress || getEnv(ENV_OPTIONS.orderbookAddress); + cmdOptions.subgraph = cmdOptions.subgraph || getEnv(ENV_OPTIONS.subgraph); + cmdOptions.lps = cmdOptions.lps || getEnv(ENV_OPTIONS.lps); + cmdOptions.gasCoverage = cmdOptions.gasCoverage || getEnv(ENV_OPTIONS.gasCoverage); + cmdOptions.orderHash = cmdOptions.orderHash || getEnv(ENV_OPTIONS.orderHash); + cmdOptions.orderOwner = cmdOptions.orderOwner || getEnv(ENV_OPTIONS.orderOwner); + cmdOptions.sleep = cmdOptions.sleep || getEnv(ENV_OPTIONS.sleep); + cmdOptions.maxRatio = cmdOptions.maxRatio || getEnv(ENV_OPTIONS.maxRatio); + cmdOptions.flashbotRpc = cmdOptions.flashbotRpc || getEnv(ENV_OPTIONS.flashbotRpc); + cmdOptions.timeout = cmdOptions.timeout || getEnv(ENV_OPTIONS.timeout); + cmdOptions.hops = cmdOptions.hops || getEnv(ENV_OPTIONS.hops); + cmdOptions.retries = cmdOptions.retries || getEnv(ENV_OPTIONS.retries); + cmdOptions.poolUpdateInterval = + cmdOptions.poolUpdateInterval || getEnv(ENV_OPTIONS.poolUpdateInterval); + cmdOptions.walletCount = cmdOptions.walletCount || getEnv(ENV_OPTIONS.walletCount); + cmdOptions.topupAmount = cmdOptions.topupAmount || getEnv(ENV_OPTIONS.topupAmount); + cmdOptions.selfFundOrders = cmdOptions.selfFundOrders || getEnv(ENV_OPTIONS.selfFundOrders); + cmdOptions.botMinBalance = cmdOptions.botMinBalance || getEnv(ENV_OPTIONS.botMinBalance); + cmdOptions.ownerProfile = cmdOptions.ownerProfile || getEnv(ENV_OPTIONS.ownerProfile); + cmdOptions.bundle = cmdOptions.bundle ? getEnv(ENV_OPTIONS.bundle) : false; if (cmdOptions.ownerProfile) { const profiles: Record = {}; cmdOptions.ownerProfile.forEach((v: string) => { @@ -735,3 +738,12 @@ export const main = async (argv: any, version?: string) => { await exporter.shutdown(); await sleep(10000); }; + +function getEnv(value: any): any { + if (value !== undefined && value !== null) { + if (typeof value === "string") { + if (value !== "" && !/^\s+$/.test(value)) return value; + } + } + return undefined; +} diff --git a/src/watcher.ts b/src/watcher.ts index dfa61916..665997ba 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -11,6 +11,7 @@ import { OwnersProfileMap, OrderbooksOwnersProfileMap, } from "./types"; +import { errorSnapshot } from "./error"; type OrderEventLog = { sender: `0x${string}`; @@ -98,16 +99,36 @@ export function watchOrderbook( onLogs: (logs) => { logs.forEach((log) => { if (log) { - watchedOrderbookOrders.orderLogs.push({ - type: log.eventName === "AddOrderV2" ? "add" : "remove", - logIndex: log.logIndex, - block: Number(log.blockNumber), - txHash: log.transactionHash, - order: logToOrder(log.args as any as OrderEventLog), - }); + try { + watchedOrderbookOrders.orderLogs.push({ + type: log.eventName === "AddOrderV2" ? "add" : "remove", + logIndex: log.logIndex, + block: Number(log.blockNumber), + txHash: log.transactionHash, + order: logToOrder(log.args as any as OrderEventLog), + }); + } catch (error) { + // eslint-disable-next-line no-console + console.warn( + errorSnapshot( + `Failed to handle orderbook ${orderbook} new event log`, + error, + ), + ); + // eslint-disable-next-line no-console + console.log("\nOriginal log:\n", log); + } } }); }, + onError: (error) => + // eslint-disable-next-line no-console + console.warn( + errorSnapshot( + `An error occured during watching new events logs of orderbook ${orderbook}`, + error, + ), + ), }); } From 2bf3aab716d4a8cbe13e8d53f56eaaf0ac2359bd Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 28 Oct 2024 14:19:00 +0000 Subject: [PATCH 13/32] Update cli.ts --- src/cli.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 88d2c948..47d161d3 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -742,8 +742,9 @@ export const main = async (argv: any, version?: string) => { function getEnv(value: any): any { if (value !== undefined && value !== null) { if (typeof value === "string") { - if (value !== "" && !/^\s+$/.test(value)) return value; - } + if (value !== "" && !/^\s*$/.test(value)) return value; + else return undefined; + } else return value; } return undefined; } From 2f1ae583a0b52395aaf3f09fec638f1dcfd4470e Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 29 Oct 2024 02:22:10 +0000 Subject: [PATCH 14/32] update --- .eslintignore | 1 + README.md | 8 +- example.env | 7 +- lib/sushiswap | 2 +- src/account.ts | 20 +-- src/cli.ts | 26 +++- src/config.ts | 328 ++++++++++++++++--------------------------- src/index.ts | 36 +++-- src/order.ts | 27 +--- src/processOrders.ts | 18 ++- src/types.ts | 9 +- src/utils.ts | 47 +++++++ test/account.test.js | 2 + test/data.js | 1 - test/e2e/e2e.test.js | 3 - test/options.test.js | 1 - test/orders.test.js | 157 +++++++++++++++------ 17 files changed, 373 insertions(+), 320 deletions(-) diff --git a/.eslintignore b/.eslintignore index 1b3e48a4..1f480b26 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,3 +12,4 @@ cache contracts docs dist +lib \ No newline at end of file diff --git a/README.md b/README.md index 8e31fccd..21992a59 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Other optional arguments are: - `-w` or `--wallet-count`, Number of wallet to submit transactions with, requirs `--mnemonic`. Will override the 'WALLET_COUNT' in env variables - `-t` or `--topup-amount`, The initial topup amount of excess wallets, requirs `--mnemonic`. Will override the 'TOPUP_AMOUNT' in env variables - `--owner-profile`, Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables +- `--public-rpc`, Allows to use public RPCs as fallbacks, default is false. Will override the 'PUBLIC_RPC' in env variables - `-V` or `--version`, output the version number - `-h` or `--help`, output usage information @@ -155,8 +156,8 @@ MNEMONIC="" # for specifying more than 1 RPC in the env, separate them by a comma and a space RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.com/polygon/{API_KEY}" -# Option to submit transactions using the flashbot RPC. -FLASHBOT_RPC="" +# Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. +WRITE_RPC="" # arb contract address ARB_ADDRESS="0x123..." @@ -225,6 +226,9 @@ SELF_FUND_ORDERS= # Specifies the owner limit, in form of owner1=limit,owner2=limit,... , example: 0x123456=12,0x3456=44 OWNER_PROFILE= + +# Allows to use public RPCs as fallbacks, default is false +PUBLIC_RPC= ``` If both env variables and CLI argument are set, the CLI arguments will be prioritized and override the env variables. diff --git a/example.env b/example.env index d2b41cc1..4151334f 100644 --- a/example.env +++ b/example.env @@ -11,8 +11,8 @@ MNEMONIC="" # for specifying more than 1 RPC in the env, separate them by a comma and a space RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.com/polygon/{API_KEY}" -# Option to submit transactions using the flashbot RPC. -FLASHBOT_RPC="" +# Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. +WRITE_RPC="" # arb contract address ARB_ADDRESS="0x123..." @@ -82,6 +82,9 @@ SELF_FUND_ORDERS= # Specifies the owner limit, in form of owner1=limit,owner2=limit,... , example: 0x123456=12,0x3456=44 OWNER_PROFILE= +# Allows to use public RPCs as fallbacks, default is false +PUBLIC_RPC= + # test rpcs vars TEST_POLYGON_RPC= TEST_BASE_RPC= diff --git a/lib/sushiswap b/lib/sushiswap index 72289f46..0f62029c 160000 --- a/lib/sushiswap +++ b/lib/sushiswap @@ -1 +1 @@ -Subproject commit 72289f46a36ee3779a215e909518214897a99912 +Subproject commit 0f62029cda417f2395dde30a26bfc192dca64c6b diff --git a/src/account.ts b/src/account.ts index a18536f9..b16980c8 100644 --- a/src/account.ts +++ b/src/account.ts @@ -36,7 +36,7 @@ export async function initAccounts( const mainAccount = await createViemClient( config.chain.id as ChainId, config.rpc, - undefined, + config.publicRpc, isMnemonic ? mnemonicToAccount(mnemonicOrPrivateKey, { addressIndex: MainAccountDerivationIndex }) : privateKeyToAccount( @@ -56,7 +56,7 @@ export async function initAccounts( await createViemClient( config.chain.id as ChainId, config.rpc, - undefined, + config.publicRpc, mnemonicToAccount(mnemonicOrPrivateKey, { addressIndex }), config.timeout, (config as any).testClientViem, @@ -203,7 +203,7 @@ export async function manageAccounts( const acc = await createViemClient( config.chain.id as ChainId, config.rpc, - undefined, + config.publicRpc, mnemonicToAccount(options.mnemonic!, { addressIndex: ++lastIndex }), config.timeout, (config as any).testClientViem, @@ -326,7 +326,7 @@ export async function rotateProviders(config: BotConfig, resetDataFetcher = true const viemClient = await createViemClient( config.chain.id as ChainId, config.rpc, - false, + config.publicRpc, undefined, config.timeout, ); @@ -335,7 +335,7 @@ export async function rotateProviders(config: BotConfig, resetDataFetcher = true config.dataFetcher = await getDataFetcher( viemClient as any as PublicClient, config.lps, - false, + config.publicRpc, ); } else { config.dataFetcher.web3Client = viemClient as any as PublicClient; @@ -348,7 +348,7 @@ export async function rotateProviders(config: BotConfig, resetDataFetcher = true const mainAcc = await createViemClient( config.chain.id as ChainId, config.rpc, - false, + config.publicRpc, config.mainAccount.account, config.timeout, ); @@ -364,7 +364,7 @@ export async function rotateProviders(config: BotConfig, resetDataFetcher = true const acc = await createViemClient( config.chain.id as ChainId, config.rpc, - false, + config.publicRpc, config.accounts[i].account, config.timeout, ); @@ -374,7 +374,11 @@ export async function rotateProviders(config: BotConfig, resetDataFetcher = true } } else { if (resetDataFetcher) { - config.dataFetcher = await getDataFetcher(config.viemClient, config.lps, false); + config.dataFetcher = await getDataFetcher( + config.viemClient, + config.lps, + config.publicRpc, + ); } } } diff --git a/src/cli.ts b/src/cli.ts index 47d161d3..3498a7ee 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -55,8 +55,8 @@ const ENV_OPTIONS = { sleep: process?.env?.SLEEP, maxRatio: process?.env?.MAX_RATIO?.toLowerCase() === "true" ? true : false, bundle: process?.env?.NO_BUNDLE?.toLowerCase() === "true" ? false : true, + publicRpc: process?.env?.PUBLIC_RPC?.toLowerCase() === "true" ? true : false, timeout: process?.env?.TIMEOUT, - flashbotRpc: process?.env?.FLASHBOT_RPC, hops: process?.env?.HOPS, retries: process?.env?.RETRIES, poolUpdateInterval: process?.env?.POOL_UPDATE_INTERVAL || "15", @@ -70,6 +70,9 @@ const ENV_OPTIONS = { rpc: process?.env?.RPC_URL ? Array.from(process?.env?.RPC_URL.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, + writeRpc: process?.env?.WRITE_RPC + ? Array.from(process?.env?.WRITE_RPC.matchAll(/[^,\s]+/g)).map((v) => v[0]) + : undefined, subgraph: process?.env?.SUBGRAPH ? Array.from(process?.env?.SUBGRAPH.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, @@ -126,8 +129,8 @@ const getOptions = async (argv: any, version?: string) => { "Seconds to wait between each arb round, default is 10, Will override the 'SLEPP' in env variables", ) .option( - "--flashbot-rpc ", - "Optional flashbot rpc url to submit transaction to, Will override the 'FLASHBOT_RPC' in env variables", + "--write-rpc ", + "Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables", ) .option( "--timeout ", @@ -173,6 +176,10 @@ const getOptions = async (argv: any, version?: string) => { "--owner-profile ", "Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables", ) + .option( + "--public-rpc", + "Allows to use public RPCs as fallbacks, default is false. Will override the 'PUBLIC_RPC' in env variables", + ) .description( [ "A NodeJS app to find and take arbitrage trades for Rain Orderbook orders against some DeFi liquidity providers, requires NodeJS v18 or higher.", @@ -189,6 +196,7 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.key = cmdOptions.key || getEnv(ENV_OPTIONS.key); cmdOptions.mnemonic = cmdOptions.mnemonic || getEnv(ENV_OPTIONS.mnemonic); cmdOptions.rpc = cmdOptions.rpc || getEnv(ENV_OPTIONS.rpc); + cmdOptions.writeRpc = cmdOptions.writeRpc || getEnv(ENV_OPTIONS.writeRpc); cmdOptions.arbAddress = cmdOptions.arbAddress || getEnv(ENV_OPTIONS.arbAddress); cmdOptions.genericArbAddress = cmdOptions.genericArbAddress || getEnv(ENV_OPTIONS.genericArbAddress); @@ -201,7 +209,6 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.orderOwner = cmdOptions.orderOwner || getEnv(ENV_OPTIONS.orderOwner); cmdOptions.sleep = cmdOptions.sleep || getEnv(ENV_OPTIONS.sleep); cmdOptions.maxRatio = cmdOptions.maxRatio || getEnv(ENV_OPTIONS.maxRatio); - cmdOptions.flashbotRpc = cmdOptions.flashbotRpc || getEnv(ENV_OPTIONS.flashbotRpc); cmdOptions.timeout = cmdOptions.timeout || getEnv(ENV_OPTIONS.timeout); cmdOptions.hops = cmdOptions.hops || getEnv(ENV_OPTIONS.hops); cmdOptions.retries = cmdOptions.retries || getEnv(ENV_OPTIONS.retries); @@ -213,6 +220,7 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.botMinBalance = cmdOptions.botMinBalance || getEnv(ENV_OPTIONS.botMinBalance); cmdOptions.ownerProfile = cmdOptions.ownerProfile || getEnv(ENV_OPTIONS.ownerProfile); cmdOptions.bundle = cmdOptions.bundle ? getEnv(ENV_OPTIONS.bundle) : false; + cmdOptions.publicRpc = cmdOptions.publicRpc || getEnv(ENV_OPTIONS.publicRpc); if (cmdOptions.ownerProfile) { const profiles: Record = {}; cmdOptions.ownerProfile.forEach((v: string) => { @@ -347,6 +355,14 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? if (!/^(0x)?[a-fA-F0-9]{64}$/.test(options.key)) throw "invalid wallet private key"; } if (!options.rpc) throw "undefined RPC URL"; + if (options.writeRpc) { + if ( + !Array.isArray(options.writeRpc) || + options.writeRpc.some((v) => typeof v !== "string") + ) { + throw `Invalid write rpcs: ${options.writeRpc}`; + } + } if (!options.arbAddress) throw "undefined arb contract address"; if (options.sleep) { if (/^[0-9]+$/.test(options.sleep)) roundGap = Number(options.sleep) * 1000; @@ -398,7 +414,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? tracer, ctx, ); - const blockNumber = (await getblockNumber(config.viemClient as any as ViemClient)) ?? 0n; + const blockNumber = (await getblockNumber(config.viemClient as any as ViemClient)) ?? 1n; return { roundGap, diff --git a/src/config.ts b/src/config.ts index 1bb44166..eb7930df 100644 --- a/src/config.ts +++ b/src/config.ts @@ -69,18 +69,17 @@ export async function createViemClient( timeout?: number, testClient?: any, ): Promise { - const transport = - !rpcs || rpcs?.length === 0 - ? fallback(fallbacks[chainId].transport, { rank: false, retryCount: 6 }) - : useFallbacks - ? fallback( - [...rpcs.map((v) => http(v, { timeout })), ...fallbacks[chainId].transport], - { rank: false, retryCount: 6 }, - ) - : fallback( - rpcs.map((v) => http(v, { timeout })), - { rank: false, retryCount: 6 }, - ); + const configuration = { rank: false, retryCount: 6 }; + const urls = rpcs?.filter((v) => typeof v === "string") ?? []; + const topRpcs = urls.map((v) => http(v, { timeout })); + const fallbacks = (fallbackRpcs[chainId] ?? []) + .filter((v) => !urls.includes(v)) + .map((v) => http(v, { timeout })); + const transport = !topRpcs.length + ? fallback(fallbacks, configuration) + : useFallbacks + ? fallback([...topRpcs, ...fallbacks], configuration) + : fallback(topRpcs, configuration); return testClient ? ((await testClient({ account })) @@ -227,197 +226,116 @@ export async function getMetaInfo(config: BotConfig, sg: string[]): Promise = { - [ChainId.ARBITRUM_NOVA]: { - transport: http("https://nova.arbitrum.io/rpc"), - liquidityProviders: ["sushiswapv3", "sushiswapv2"], - }, - [ChainId.ARBITRUM]: { - transport: [ - http( - "https://lb.drpc.org/ogrpc?network=arbitrum&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", - ), - http("https://rpc.ankr.com/arbitrum"), - http("https://arbitrum-one.public.blastapi.io"), - http("https://endpoints.omniatech.io/v1/arbitrum/one/public"), - http("https://arb1.croswap.com/rpc"), - http("https://1rpc.io/arb"), - http("https://arbitrum.blockpi.network/v1/rpc/public"), - http("https://arb-mainnet-public.unifra.io"), - ], - liquidityProviders: ["dfyn", "elk", "sushiswapv3", "uniswapv3", "sushiswapv2", "camelot"], - }, - [ChainId.AVALANCHE]: { - transport: [ - http("https://api.avax.network/ext/bc/C/rpc"), - http("https://rpc.ankr.com/avalanche"), - ], - liquidityProviders: ["elk", "traderjoe", "sushiswapv3", "sushiswapv2"], - }, - [ChainId.BOBA]: { - transport: [ - http("https://mainnet.boba.network"), - http("https://lightning-replica.boba.network"), - ], - liquidityProviders: ["sushiswapv3", "sushiswapv2"], - }, - [ChainId.BOBA_AVAX]: { - transport: [http("https://avax.boba.network"), http("https://replica.avax.boba.network")], - liquidityProviders: ["sushiswapv2"], - }, - [ChainId.BOBA_BNB]: { - transport: [http("https://bnb.boba.network"), http("https://replica.bnb.boba.network")], - liquidityProviders: ["sushiswapv2"], - }, - [ChainId.BSC]: { - transport: [ - http("https://rpc.ankr.com/bsc"), - http( - "https://lb.drpc.org/ogrpc?network=bsc&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", - ), - http("https://bsc-dataseed.binance.org"), - http("https://bsc-dataseed1.binance.org"), - http("https://bsc-dataseed2.binance.org"), - ], - liquidityProviders: [ - "apeswap", - "biswap", - "elk", - "jetswap", - "pancakeswap", - "sushiswapv3", - "sushiswapv2", - "uniswapv3", - ], - }, - [ChainId.BTTC]: { - transport: http("https://rpc.bittorrentchain.io"), - }, - [ChainId.CELO]: { - transport: http("https://forno.celo.org"), - liquidityProviders: ["ubeswap", "sushiswapv2"], - }, - [ChainId.ETHEREUM]: { - transport: [ - http( - "https://lb.drpc.org/ogrpc?network=ethereum&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", - ), - http("https://eth.llamarpc.com"), - // http('https://eth.rpc.blxrbdn.com'), - // http('https://virginia.rpc.blxrbdn.com'), - // http('https://singapore.rpc.blxrbdn.com'), - // http('https://uk.rpc.blxrbdn.com'), - http("https://1rpc.io/eth"), - http("https://ethereum.publicnode.com"), - http("https://cloudflare-eth.com"), - ], - liquidityProviders: [ - "apeswap", - "curveswap", - "elk", - "pancakeswap", - "sushiswapv3", - "sushiswapv2", - "uniswapv2", - "uniswapv3", - ], - }, - [ChainId.FANTOM]: { - transport: [ - http("https://rpc.ankr.com/fantom"), - http("https://rpc.fantom.network"), - http("https://rpc2.fantom.network"), - ], - liquidityProviders: ["dfyn", "elk", "jetswap", "spookyswap", "sushiswapv3", "sushiswapv2"], - }, - [ChainId.FUSE]: { - transport: http("https://rpc.fuse.io"), - liquidityProviders: ["elk", "sushiswapv3", "sushiswapv2"], - }, - [ChainId.GNOSIS]: { - transport: http("https://rpc.ankr.com/gnosis"), - liquidityProviders: ["elk", "honeyswap", "sushiswapv3", "sushiswapv2"], - }, - [ChainId.HARMONY]: { - transport: [http("https://api.harmony.one"), http("https://rpc.ankr.com/harmony")], - liquidityProviders: ["sushiswapv2"], - }, - [ChainId.KAVA]: { - transport: [http("https://evm.kava.io"), http("https://evm2.kava.io")], - liquidityProviders: ["elk"], - }, - [ChainId.MOONBEAM]: { - transport: [ - http("https://rpc.api.moonbeam.network"), - http("https://rpc.ankr.com/moonbeam"), - ], - liquidityProviders: ["sushiswapv2"], - }, - [ChainId.MOONRIVER]: { - transport: http("https://rpc.api.moonriver.moonbeam.network"), - liquidityProviders: ["elk", "sushiswapv3", "sushiswapv2"], - }, - [ChainId.OPTIMISM]: { - transport: [ - http( - "https://lb.drpc.org/ogrpc?network=optimism&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", - ), - http("https://rpc.ankr.com/optimism"), - http("https://optimism-mainnet.public.blastapi.io"), - http("https://1rpc.io/op"), - http("https://optimism.blockpi.network/v1/rpc/public"), - http("https://mainnet.optimism.io"), - ], - liquidityProviders: ["elk", "sushiswapv3", "uniswapv3"], - }, - [ChainId.POLYGON]: { - transport: [ - http("https://polygon.llamarpc.com"), - // http('https://polygon.rpc.blxrbdn.com'), - http("https://polygon-mainnet.public.blastapi.io"), - http("https://polygon.blockpi.network/v1/rpc/public"), - http("https://polygon-rpc.com"), - http("https://rpc.ankr.com/polygon"), - http("https://matic-mainnet.chainstacklabs.com"), - http("https://polygon-bor.publicnode.com"), - http("https://rpc-mainnet.matic.quiknode.pro"), - http("https://rpc-mainnet.maticvigil.com"), - // ...polygon.rpcUrls.default.http.map((url) => http(url)), - ], - liquidityProviders: [ - "apeswap", - "dfyn", - "elk", - "jetswap", - "quickswap", - "sushiswapv3", - "sushiswapv2", - "uniswapv3", - ], - }, - [ChainId.POLYGON_ZKEVM]: { - transport: [ - http("https://zkevm-rpc.com"), - http("https://rpc.ankr.com/polygon_zkevm"), - http("https://rpc.polygon-zkevm.gateway.fm"), - ], - liquidityProviders: ["dovishv3", "sushiswapv3"], - }, - [ChainId.THUNDERCORE]: { - transport: [ - http("https://mainnet-rpc.thundercore.com"), - http("https://mainnet-rpc.thundercore.io"), - http("https://mainnet-rpc.thundertoken.net"), - ], - liquidityProviders: ["laserswap", "sushiswapv3"], - }, - // flare - 14: { - transport: [ - http("https://rpc.ankr.com/flare"), - http("https://flare-api.flare.network/ext/C/rpc"), - http("https://flare.rpc.thirdweb.com"), - ], - liquidityProviders: ["enosys", "blazeswap"], - }, +export const fallbackRpcs: Record = { + [ChainId.ARBITRUM_NOVA]: ["https://nova.arbitrum.io/rpc"], + [ChainId.ARBITRUM]: [ + "https://arbitrum.drpc.org", + "https://arb-pokt.nodies.app", + "https://1rpc.io/arb", + "https://rpc.ankr.com/arbitrum", + "https://arbitrum-one.public.blastapi.io", + "https://endpoints.omniatech.io/v1/arbitrum/one/public", + "https://arb1.croswap.com/rpc", + "https://arbitrum.blockpi.network/v1/rpc/public", + "https://arb-mainnet-public.unifra.io", + "https://lb.drpc.org/ogrpc?network=arbitrum&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", + ], + [ChainId.AVALANCHE]: [ + "https://api.avax.network/ext/bc/C/rpc", + "https://rpc.ankr.com/avalanche", + ], + [ChainId.BOBA]: ["https://mainnet.boba.network", "https://lightning-replica.boba.network"], + [ChainId.BOBA_AVAX]: ["https://avax.boba.network", "https://replica.avax.boba.network"], + [ChainId.BOBA_BNB]: ["https://bnb.boba.network", "https://replica.bnb.boba.network"], + [ChainId.BSC]: [ + "https://rpc.ankr.com/bsc", + "https://bsc.blockpi.network/v1/rpc/public", + "https://bsc-pokt.nodies.app", + "https://bscrpc.com", + "https://1rpc.io/bnb", + "https://bsc.drpc.org", + "https://bsc.meowrpc.com", + "https://binance.llamarpc.com", + "https://bsc-dataseed.binance.org", + "https://bsc-dataseed1.binance.org", + "https://bsc-dataseed2.binance.org", + "https://lb.drpc.org/ogrpc?network=bsc&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", + ], + [ChainId.BTTC]: ["https://rpc.bittorrentchain.io"], + [ChainId.CELO]: ["https://forno.celo.org"], + [ChainId.ETHEREUM]: [ + "https://eth-pokt.nodies.app", + "https://eth.drpc.org", + "https://ethereum-rpc.publicnode.com", + "https://eth.llamarpc.com", + "https://1rpc.io/eth", + "https://ethereum.publicnode.com", + "https://cloudflare-eth.com", + "https://lb.drpc.org/ogrpc?network=ethereum&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", + ], + [ChainId.FANTOM]: [ + "https://rpc.ankr.com/fantom", + "https://rpc.fantom.network", + "https://rpc2.fantom.network", + ], + [ChainId.FUSE]: ["https://rpc.fuse.io"], + [ChainId.GNOSIS]: ["https://rpc.ankr.com/gnosis"], + [ChainId.HARMONY]: ["https://api.harmony.one", "https://rpc.ankr.com/harmony"], + [ChainId.KAVA]: ["https://evm.kava.io", "https://evm2.kava.io"], + [ChainId.MOONBEAM]: ["https://rpc.api.moonbeam.network", "https://rpc.ankr.com/moonbeam"], + [ChainId.MOONRIVER]: ["https://rpc.api.moonriver.moonbeam.network"], + [ChainId.OPTIMISM]: [ + "https://rpc.ankr.com/optimism", + "https://optimism-mainnet.public.blastapi.io", + "https://1rpc.io/op", + "https://optimism.blockpi.network/v1/rpc/public", + "https://mainnet.optimism.io", + "https://lb.drpc.org/ogrpc?network=optimism&dkey=Ak765fp4zUm6uVwKu4annC8M80dnCZkR7pAEsm6XXi_w", + ], + [ChainId.POLYGON]: [ + "https://polygon.meowrpc.com", + "https://polygon-rpc.com", + "https://polygon-pokt.nodies.app", + "https://polygon-bor-rpc.publicnode.com", + "https://1rpc.io/matic", + "https://polygon-mainnet.public.blastapi.io", + "https://polygon.blockpi.network/v1/rpc/public", + "https://polygon.llamarpc.com", + "https://polygon-rpc.com", + "https://rpc.ankr.com/polygon", + "https://matic-mainnet.chainstacklabs.com", + "https://polygon-bor.publicnode.com", + "https://rpc-mainnet.matic.quiknode.pro", + "https://rpc-mainnet.maticvigil.com", + ], + [ChainId.POLYGON_ZKEVM]: [ + "https://zkevm-rpc.com", + "https://rpc.ankr.com/polygon_zkevm", + "https://rpc.polygon-zkevm.gateway.fm", + ], + [ChainId.THUNDERCORE]: [ + "https://mainnet-rpc.thundercore.com", + "https://mainnet-rpc.thundercore.io", + "https://mainnet-rpc.thundertoken.net", + ], + [ChainId.FLARE]: [ + "https://rpc.ankr.com/flare", + "https://flare-api.flare.network/ext/C/rpc", + "https://flare.rpc.thirdweb.com", + ], + [ChainId.LINEA]: [ + "https://linea.blockpi.network/v1/rpc/public", + "https://rpc.linea.build", + "https://linea-rpc.publicnode.com", + "https://1rpc.io/linea", + "https://linea.drpc.org", + ], + [ChainId.BASE]: [ + "https://base-rpc.publicnode.com", + "https://base.blockpi.network/v1/rpc/public", + "https://1rpc.io/base", + "https://base-pokt.nodies.app", + "https://mainnet.base.org", + "https://base.meowrpc.com", + ], } as const; diff --git a/src/index.ts b/src/index.ts index d1cac058..e9abe991 100644 --- a/src/index.ts +++ b/src/index.ts @@ -88,13 +88,12 @@ export async function getConfig( tracer?: Tracer, ctx?: Context, ): Promise { - const AddressPattern = /^0x[a-fA-F0-9]{40}$/; - if (!AddressPattern.test(arbAddress)) throw "invalid arb contract address"; - if (options.genericArbAddress && !AddressPattern.test(options.genericArbAddress)) { + if (!ethers.utils.isAddress(arbAddress)) throw "invalid arb contract address"; + if (options.genericArbAddress && !ethers.utils.isAddress(options.genericArbAddress)) { throw "invalid generic arb contract address"; } - let timeout = 30_000; + let timeout = 15_000; if (options.timeout) { if (typeof options.timeout === "number") { if (!Number.isInteger(options.timeout) || options.timeout == 0) @@ -140,19 +139,32 @@ export async function getConfig( const chainId = (await getChainId(rpcUrls)) as ChainId; const config = getChainConfig(chainId) as any as BotConfig; const lps = processLps(options.lps); - const viemClient = await createViemClient(chainId, rpcUrls, false, undefined, options.timeout); - const watchClient = await createViemClient(chainId, rpcUrls, false, undefined, options.timeout); - const dataFetcher = await getDataFetcher(viemClient as any as PublicClient, lps, false); + const viemClient = await createViemClient( + chainId, + rpcUrls, + options.publicRpc, + undefined, + options.timeout, + ); + const watchClient = await createViemClient( + chainId, + rpcUrls, + options.publicRpc, + undefined, + options.timeout, + ); + const dataFetcher = await getDataFetcher( + viemClient as any as PublicClient, + lps, + options.publicRpc, + ); if (!config) throw `Cannot find configuration for the network with chain id: ${chainId}`; - config.bundle = true; - if (options.bundle !== undefined) config.bundle = !!options.bundle; - config.rpc = rpcUrls; config.arbAddress = arbAddress; config.genericArbAddress = options.genericArbAddress; config.timeout = timeout; - config.flashbotRpc = options.flashbotRpc; + config.writeRpc = options.writeRpc; config.maxRatio = !!options.maxRatio; config.hops = hops; config.retries = retries; @@ -163,6 +175,8 @@ export async function getConfig( config.watchedTokens = options.tokens ?? []; config.selfFundOrders = options.selfFundOrders; config.watchClient = watchClient; + config.publicRpc = options.publicRpc; + config.walletKey = walletKey; // init accounts const { mainAccount, accounts } = await initAccounts(walletKey, config, options, tracer, ctx); diff --git a/src/order.ts b/src/order.ts index 628330fa..67cde083 100644 --- a/src/order.ts +++ b/src/order.ts @@ -1,8 +1,8 @@ +import { OrderV3 } from "./abis"; import { SgOrder } from "./query"; import { toOrder } from "./watcher"; -import { shuffleArray, sleep } from "./utils"; -import { erc20Abi, OrderV3 } from "./abis"; -import { decodeAbiParameters, parseAbi, parseAbiParameters } from "viem"; +import { getTokenSymbol, shuffleArray } from "./utils"; +import { decodeAbiParameters, parseAbiParameters } from "viem"; import { Pair, Order, @@ -280,24 +280,3 @@ function gatherPairs( } } } - -/** - * Get token symbol - * @param address - The address of token - * @param viemClient - The viem client - */ -export async function getTokenSymbol(address: string, viemClient: ViemClient): Promise { - // 3 retries - for (let i = 0; i < 3; i++) { - try { - return await viemClient.readContract({ - address: address as `0x${string}`, - abi: parseAbi(erc20Abi), - functionName: "symbol", - }); - } catch { - await sleep(10_000); - } - } - return "UnknownSymbol"; -} diff --git a/src/processOrders.ts b/src/processOrders.ts index 5966ff82..5a59a844 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -178,15 +178,19 @@ export const processOrders = async ( takeOrders: [pairOrders.takeOrders[i]], }; const signer = accounts.length ? accounts[0] : mainAccount; - const flashbotSigner = config.flashbotRpc + const writeSigner = config.writeRpc ? await createViemClient( config.chain.id as ChainId, - [config.flashbotRpc], - undefined, + config.writeRpc, + false, privateKeyToAccount( - ethers.utils.hexlify( - signer.account.getHdKey().privateKey!, - ) as `0x${string}`, + signer.account.getHdKey + ? (ethers.utils.hexlify( + signer.account.getHdKey().privateKey!, + ) as `0x${string}`) + : ((config.walletKey.startsWith("0x") + ? config.walletKey + : "0x" + config.walletKey) as `0x${string}`), ), config.timeout, ) @@ -205,7 +209,7 @@ export const processOrders = async ( viemClient, dataFetcher, signer, - flashbotSigner, + flashbotSigner: writeSigner, arb, genericArb, orderbook, diff --git a/src/types.ts b/src/types.ts index dcc459c9..c571f3a0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,6 +23,7 @@ export type CliOptions = { key?: string; mnemonic?: string; rpc: string[]; + writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; orderbookAddress?: string; @@ -33,7 +34,6 @@ export type CliOptions = { orderOwner?: string; sleep: number; maxRatio: boolean; - flashbotRpc?: string; timeout?: number; hops: number; retries: number; @@ -41,10 +41,10 @@ export type CliOptions = { walletCount?: number; topupAmount?: string; botMinBalance: string; - bundle: boolean; selfFundOrders?: SelfFundOrder[]; tokens?: TokenDetails[]; ownerProfile?: Record; + publicRpc: boolean; }; export type TokenDetails = { @@ -148,15 +148,14 @@ export type BotConfig = { key?: string; mnemonic?: string; rpc: string[]; + writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; lps: LiquidityProviders[]; maxRatio: boolean; - flashbotRpc?: string; timeout?: number; hops: number; retries: number; - bundle: boolean; gasCoveragePercentage: string; watchedTokens?: TokenDetails[]; viemClient: PublicClient; @@ -165,6 +164,8 @@ export type BotConfig = { accounts: ViemClient[]; selfFundOrders?: SelfFundOrder[]; watchClient: ViemClient; + publicRpc: boolean; + walletKey: string; }; export type Report = { diff --git a/src/utils.ts b/src/utils.ts index ad6cd4da..b867a69c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -848,6 +848,9 @@ export async function quoteSingleOrder( } } +/** + * Get a TakeOrder type consumable by orderbook Quote lib for quoting orders + */ export function getQuoteConfig(orderDetails: any): TakeOrder { return { order: { @@ -1082,6 +1085,11 @@ export const getRpSwap = async ( } }; +/** + * Gets all distinct tokens of all the orders' IOs from a subgraph query, + * used to to keep a cache of known tokens at runtime to not fetch their + * details everytime with onchain calls + */ export function getOrdersTokens(ordersDetails: SgOrder[]): TokenDetails[] { const tokens: TokenDetails[] = []; for (let i = 0; i < ordersDetails.length; i++) { @@ -1121,6 +1129,9 @@ export function getOrdersTokens(ordersDetails: SgOrder[]): TokenDetails[] { return tokens; } +/** + * Checks if a route exists between 2 tokens using sushi router + */ export async function routeExists( config: BotConfig, fromToken: Token, @@ -1144,6 +1155,9 @@ export async function routeExists( } } +/** + * Json serializer function for handling bigint type + */ export function withBigintSerializer(_k: string, v: any) { if (typeof v == "bigint") { return v.toString(); @@ -1244,11 +1258,17 @@ export async function checkOwnedOrders( return result; } +/** + * Converts to a float number + */ export function toNumber(value: BigNumberish): number { const valueString = ethers.utils.formatUnits(value); return Number.parseFloat(valueString.substring(0, valueString.includes(".") ? 18 : 17)); } +/** + * Get market quote (price) for a token pair using sushi router + */ export function getMarketQuote( config: BotConfig, fromToken: Token, @@ -1282,6 +1302,9 @@ export function getMarketQuote( } } +/** + * Checks if an a value is a big numberish, from ethers + */ export function isBigNumberish(value: any): value is BigNumberish { return ( value != null && @@ -1294,6 +1317,9 @@ export function isBigNumberish(value: any): value is BigNumberish { ); } +/** + * Get block number with retries, using viem client + */ export async function getblockNumber(viemClient: ViemClient): Promise { for (let i = 0; i < 3; i++) { try { @@ -1304,3 +1330,24 @@ export async function getblockNumber(viemClient: ViemClient): Promise { + // 3 retries + for (let i = 0; i < 3; i++) { + try { + return await viemClient.readContract({ + address: address as `0x${string}`, + abi: parseAbi(erc20Abi), + functionName: "symbol", + }); + } catch { + await sleep(5_000); + } + } + return "UnknownSymbol"; +} diff --git a/test/account.test.js b/test/account.test.js index 5deb07a9..3f235210 100644 --- a/test/account.test.js +++ b/test/account.test.js @@ -193,6 +193,7 @@ describe("Test accounts", async function () { mainAccount, accounts, dataFetcher: dataFectherBefore, + publicRpc: false, }; await rotateProviders(config, mainAccount, true); @@ -203,6 +204,7 @@ describe("Test accounts", async function () { assert.exists(config.viemClient); assert.exists(config.dataFetcher); assert.equal(config.chain.id, 137); + assert.equal(config.viemClient.transport.transports.length, 2); assert.equal(config.viemClient.transport.transports[0].value.url, config.rpc[0]); assert.equal(config.viemClient.transport.transports[1].value.url, config.rpc[1]); assert.equal(config.mainAccount.provider, config.provider); diff --git a/test/data.js b/test/data.js index fa7e5897..b7ba410f 100644 --- a/test/data.js +++ b/test/data.js @@ -49,7 +49,6 @@ const toToken = new Token({ const scannerUrl = "https://scanner.com"; const config = { hops: 3, - bundle: false, retries: 2, maxRatio: true, concurrency: "max", diff --git a/test/e2e/e2e.test.js b/test/e2e/e2e.test.js index 29b8a120..ae725205 100644 --- a/test/e2e/e2e.test.js +++ b/test/e2e/e2e.test.js @@ -237,7 +237,6 @@ for (let i = 0; i < testData.length; i++) { config.shuffle = false; config.signer = bot; config.hops = 2; - config.bundle = true; config.retries = 1; config.lps = liquidityProviders; config.rpVersion = rpVersion; @@ -570,7 +569,6 @@ for (let i = 0; i < testData.length; i++) { config.shuffle = false; config.signer = bot; config.hops = 2; - config.bundle = true; config.retries = 1; config.lps = liquidityProviders; config.rpVersion = rpVersion; @@ -923,7 +921,6 @@ for (let i = 0; i < testData.length; i++) { config.shuffle = false; config.signer = bot; config.hops = 2; - config.bundle = true; config.retries = 1; config.lps = liquidityProviders; config.rpVersion = rpVersion; diff --git a/test/options.test.js b/test/options.test.js index fb473255..a3c63b33 100644 --- a/test/options.test.js +++ b/test/options.test.js @@ -20,7 +20,6 @@ describe("Test app options", async function () { assert.equal(config.maxRatio, false); assert.equal(config.hops, 1); assert.equal(config.retries, 1); - assert.equal(config.bundle, true); assert.equal(config.chain.id, 137); assert.equal(config.gasCoveragePercentage, "100"); assert.deepEqual(config.rpc, rpcs); diff --git a/test/orders.test.js b/test/orders.test.js index d22e75cc..c9cfed3c 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -131,22 +131,14 @@ describe("Test order details", async function () { { balance: "1", vaultId: "0x01", - token: { - address: token1, - decimals: 6, - symbol: "NewToken1", - }, + token: token1, }, ], outputs: [ { balance: "1", vaultId: "0x01", - token: { - address: token2, - decimals: 18, - symbol: "NewToken2", - }, + token: token2, }, ], }; @@ -731,39 +723,72 @@ describe("Test order details", async function () { it("should prepare orders for rounds by specified owner limits", async function () { const orderbook = hexlify(randomBytes(20)).toLowerCase(); - const owner = hexlify(randomBytes(20)).toLowerCase(); - const token1 = hexlify(randomBytes(20)).toLowerCase(); - const token2 = hexlify(randomBytes(20)).toLowerCase(); - const [order1, order2, order3, order4, order5, order6] = [ - getNewOrder(orderbook, owner, token1, token2, 1), - getNewOrder(orderbook, owner, token1, token2, 2), - getNewOrder(orderbook, owner, token1, token2, 3), - getNewOrder(orderbook, owner, token1, token2, 4), - getNewOrder(orderbook, owner, token1, token2, 5), - getNewOrder(orderbook, owner, token1, token2, 6), + const owner1 = hexlify(randomBytes(20)).toLowerCase(); + const owner2 = hexlify(randomBytes(20)).toLowerCase(); + const token1 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const token2 = { + address: hexlify(randomBytes(20)).toLowerCase(), + decimals: 6, + symbol: "NewToken1", + }; + const [order1, order2, order3, order4, order5, order6, order7, order8] = [ + getNewOrder(orderbook, owner1, token1, token2, 1), // owner 1 + getNewOrder(orderbook, owner1, token1, token2, 2), // // + getNewOrder(orderbook, owner1, token1, token2, 3), // // + getNewOrder(orderbook, owner1, token1, token2, 4), // // + getNewOrder(orderbook, owner1, token1, token2, 5), // // + getNewOrder(orderbook, owner1, token1, token2, 6), // // + getNewOrder(orderbook, owner2, token2, token1, 1), // owner 2 + getNewOrder(orderbook, owner2, token2, token1, 2), // // ]; + const owner1Orders = [order1, order2, order3, order4, order5, order6]; + const owner2Orders = [order7, order8]; // build orderbook owner map const allOrders = await getOrderbookOwnersProfileMapFromSg( - [order1, order2, order3, order4, order5, order6], + [order1, order2, order3, order4, order5, order6, order7, order8], undefined, [], - { [owner]: 3 }, // set owner limit as 3 + { [owner1]: 3, [owner2]: 1 }, // set owner1 limit as 3, owner2 to 1 ); - // prepare orders for first round, ie first 3 orders should get consumed + // prepare orders for first round const result1 = prepareOrdersForRound(allOrders, false); const expected1 = [ [ { - buyToken: token1.toLowerCase(), - buyTokenSymbol: "NewToken1", - buyTokenDecimals: 6, - sellToken: token2.toLowerCase(), - sellTokenSymbol: "NewToken2", - sellTokenDecimals: 18, + buyToken: token1.address, + buyTokenSymbol: token1.symbol, + buyTokenDecimals: token1.decimals, + sellToken: token2.address, + sellTokenSymbol: token2.symbol, + sellTokenDecimals: token2.decimals, orderbook, - takeOrders: [order1, order2, order3].map((v) => ({ + // first 3 owner1 orders for round1, owner1 limit is 3 + takeOrders: owner1Orders.slice(0, 3).map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + { + buyToken: token2.address, + buyTokenSymbol: token2.symbol, + buyTokenDecimals: token2.decimals, + sellToken: token1.address, + sellTokenSymbol: token1.symbol, + sellTokenDecimals: token1.decimals, + orderbook, + // first 1 owner2 orders for round1, owner2 limit is 1 + takeOrders: owner2Orders.slice(0, 1).map((v) => ({ id: v.id, takeOrder: { order: v.struct, @@ -777,19 +802,39 @@ describe("Test order details", async function () { ]; assert.deepEqual(result1, expected1); - // prepare orders for second round, ie second 3 orders should get consumed + // prepare orders for second round const result2 = prepareOrdersForRound(allOrders, false); const expected2 = [ [ { - buyToken: token1.toLowerCase(), - buyTokenSymbol: "NewToken1", - buyTokenDecimals: 6, - sellToken: token2.toLowerCase(), - sellTokenSymbol: "NewToken2", - sellTokenDecimals: 18, + buyToken: token1.address, + buyTokenSymbol: token1.symbol, + buyTokenDecimals: token1.decimals, + sellToken: token2.address, + sellTokenSymbol: token2.symbol, + sellTokenDecimals: token2.decimals, orderbook, - takeOrders: [order4, order5, order6].map((v) => ({ + // second 3 owner1 orders for round2, owner1 limit is 3 + takeOrders: owner1Orders.slice(3, owner1Orders.length).map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + { + buyToken: token2.address, + buyTokenSymbol: token2.symbol, + buyTokenDecimals: token2.decimals, + sellToken: token1.address, + sellTokenSymbol: token1.symbol, + sellTokenDecimals: token1.decimals, + orderbook, + // second 1 owner2 orders for round2, owner2 limit is 1 + takeOrders: owner2Orders.slice(1, owner2Orders.length).map((v) => ({ id: v.id, takeOrder: { order: v.struct, @@ -804,19 +849,39 @@ describe("Test order details", async function () { assert.deepEqual(result2, expected2); // prepare orders for 3rd round, so should be back to consuming - // first 3 order again as 6 total order were consumed by first 2 rounds + // orders of onwer1 and 2 just like round 1 const result3 = prepareOrdersForRound(allOrders, false); const expected3 = [ [ { - buyToken: token1.toLowerCase(), - buyTokenSymbol: "NewToken1", - buyTokenDecimals: 6, - sellToken: token2.toLowerCase(), - sellTokenSymbol: "NewToken2", - sellTokenDecimals: 18, + buyToken: token1.address, + buyTokenSymbol: token1.symbol, + buyTokenDecimals: token1.decimals, + sellToken: token2.address, + sellTokenSymbol: token2.symbol, + sellTokenDecimals: token2.decimals, + orderbook, + // first 3 owner1 orders again for round3, owner1 limit is 3 + takeOrders: owner1Orders.slice(0, 3).map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + { + buyToken: token2.address, + buyTokenSymbol: token2.symbol, + buyTokenDecimals: token2.decimals, + sellToken: token1.address, + sellTokenSymbol: token1.symbol, + sellTokenDecimals: token1.decimals, orderbook, - takeOrders: [order1, order2, order3].map((v) => ({ + // first 1 owner2 orders again for round3, owner2 limit is 1 + takeOrders: owner2Orders.slice(0, 1).map((v) => ({ id: v.id, takeOrder: { order: v.struct, From f7915cb076e4f5326bf937f2057fe19fa2d9094c Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 29 Oct 2024 02:39:01 +0000 Subject: [PATCH 15/32] Update utils.ts --- src/utils.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/utils.ts b/src/utils.ts index b867a69c..a37a513c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1351,3 +1351,14 @@ export async function getTokenSymbol(address: string, viemClient: ViemClient): P } return "UnknownSymbol"; } + +export function memory(msg: string) { + // eslint-disable-next-line no-console + console.log(msg); + for (const [key, value] of Object.entries(process.memoryUsage())) { + // eslint-disable-next-line no-console + console.log(`Memory usage by ${key}, ${value / 1_000_000}MB `); + } + // eslint-disable-next-line no-console + console.log("\n---\n"); +} \ No newline at end of file From e17a293ac46fdb2ecb3c9452fd22917cd60fc5f4 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 29 Oct 2024 02:53:49 +0000 Subject: [PATCH 16/32] update --- src/utils.ts | 2 +- src/watcher.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index a37a513c..0aa732ef 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1361,4 +1361,4 @@ export function memory(msg: string) { } // eslint-disable-next-line no-console console.log("\n---\n"); -} \ No newline at end of file +} diff --git a/src/watcher.ts b/src/watcher.ts index 665997ba..e99f7aac 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -176,10 +176,7 @@ export async function handleOrderbooksNewLogs( const watchedOrderbookLogs = watchedOrderbooksOrders[ob]; const logs = watchedOrderbookLogs.orderLogs.splice(0); // make sure logs are sorted before applying them to the map - logs.sort((a, b) => { - const block = a.block - b.block; - return block !== 0 ? block : a.logIndex - b.logIndex; - }); + logs.sort((a, b) => a.block - b.block || a.logIndex - b.logIndex); await handleNewOrderLogs( ob, logs, From dae6ba3fca14fe39833fbfb49656b3b7f5bb0ef0 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Thu, 31 Oct 2024 18:21:27 +0000 Subject: [PATCH 17/32] update --- README.md | 6 +++++- example.env | 3 +++ src/cli.ts | 11 +++++++++++ src/config.ts | 13 +++++++++++-- src/index.ts | 5 +++-- src/types.ts | 2 ++ 6 files changed, 35 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 21992a59..4a18a02b 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,8 @@ Other optional arguments are: - `--sleep`, Seconds to wait between each arb round, default is 10, Will override the 'SLEPP' in env variables - `--max-ratio`, Option to maximize maxIORatio, Will override the 'MAX_RATIO' in env variables - `--timeout`, Optional seconds to wait for the transaction to mine before disregarding it, Will override the 'TIMEOUT' in env variables -- `--flashbot-rpc`, Optional flashbot rpc url to submit transaction to, Will override the 'FLASHBOT_RPC' in env variables +- `--write-rpc`, Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables" +- `--watch-rpc`, RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables" - `--no-bundle`, Flag for not bundling orders based on pairs and clear each order individually. Will override the 'NO_BUNDLE' in env variables - `--hops`, Option to specify how many hops the binary search should do, default is 0 if left unspecified, Will override the 'HOPS' in env variables - `--retries`, Option to specify how many retries should be done for the same order, max value is 3, default is 1 if left unspecified, Will override the 'RETRIES' in env variables @@ -159,6 +160,9 @@ RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.co # Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. WRITE_RPC="" +# RPC URLs to watch for new orders, should support required RPC methods +WATCH_RPC="" + # arb contract address ARB_ADDRESS="0x123..." diff --git a/example.env b/example.env index 4151334f..12d78f60 100644 --- a/example.env +++ b/example.env @@ -14,6 +14,9 @@ RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.co # Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. WRITE_RPC="" +# RPC URLs to watch for new orders, should support required RPC methods +WATCH_RPC="" + # arb contract address ARB_ADDRESS="0x123..." diff --git a/src/cli.ts b/src/cli.ts index 3498a7ee..d2394ab6 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -73,6 +73,9 @@ const ENV_OPTIONS = { writeRpc: process?.env?.WRITE_RPC ? Array.from(process?.env?.WRITE_RPC.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, + watchRpc: process?.env?.WATCH_RPC + ? Array.from(process?.env?.WATCH_RPC.matchAll(/[^,\s]+/g)).map((v) => v[0]) + : undefined, subgraph: process?.env?.SUBGRAPH ? Array.from(process?.env?.SUBGRAPH.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, @@ -132,6 +135,10 @@ const getOptions = async (argv: any, version?: string) => { "--write-rpc ", "Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables", ) + .option( + "--watch-rpc ", + "RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables", + ) .option( "--timeout ", "Optional seconds to wait for the transaction to mine before disregarding it, Will override the 'TIMEOUT' in env variables", @@ -197,6 +204,7 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.mnemonic = cmdOptions.mnemonic || getEnv(ENV_OPTIONS.mnemonic); cmdOptions.rpc = cmdOptions.rpc || getEnv(ENV_OPTIONS.rpc); cmdOptions.writeRpc = cmdOptions.writeRpc || getEnv(ENV_OPTIONS.writeRpc); + cmdOptions.watchRpc = cmdOptions.watchRpc || getEnv(ENV_OPTIONS.watchRpc); cmdOptions.arbAddress = cmdOptions.arbAddress || getEnv(ENV_OPTIONS.arbAddress); cmdOptions.genericArbAddress = cmdOptions.genericArbAddress || getEnv(ENV_OPTIONS.genericArbAddress); @@ -355,6 +363,9 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? if (!/^(0x)?[a-fA-F0-9]{64}$/.test(options.key)) throw "invalid wallet private key"; } if (!options.rpc) throw "undefined RPC URL"; + if (!options.watchRpc || !Array.isArray(options.watchRpc) || !options.watchRpc.length) { + throw "undefined watch RPC URL"; + } if (options.writeRpc) { if ( !Array.isArray(options.writeRpc) || diff --git a/src/config.ts b/src/config.ts index eb7930df..5c44a465 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,7 @@ import { publicActions, PublicClient, walletActions, + webSocket, } from "viem"; import { STABLES, @@ -71,10 +72,18 @@ export async function createViemClient( ): Promise { const configuration = { rank: false, retryCount: 6 }; const urls = rpcs?.filter((v) => typeof v === "string") ?? []; - const topRpcs = urls.map((v) => http(v, { timeout })); + const topRpcs = urls.map((v) => + v.startsWith("http") + ? http(v, { timeout }) + : webSocket(v, { timeout, keepAlive: true, reconnect: true }), + ); const fallbacks = (fallbackRpcs[chainId] ?? []) .filter((v) => !urls.includes(v)) - .map((v) => http(v, { timeout })); + .map((v) => + v.startsWith("http") + ? http(v, { timeout }) + : webSocket(v, { timeout, keepAlive: true, reconnect: true }), + ); const transport = !topRpcs.length ? fallback(fallbacks, configuration) : useFallbacks diff --git a/src/index.ts b/src/index.ts index e9abe991..116da308 100644 --- a/src/index.ts +++ b/src/index.ts @@ -148,8 +148,8 @@ export async function getConfig( ); const watchClient = await createViemClient( chainId, - rpcUrls, - options.publicRpc, + options.watchRpc, + false, undefined, options.timeout, ); @@ -177,6 +177,7 @@ export async function getConfig( config.watchClient = watchClient; config.publicRpc = options.publicRpc; config.walletKey = walletKey; + config.watchRpc = options.watchRpc; // init accounts const { mainAccount, accounts } = await initAccounts(walletKey, config, options, tracer, ctx); diff --git a/src/types.ts b/src/types.ts index c571f3a0..c1c6ef6b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,6 +23,7 @@ export type CliOptions = { key?: string; mnemonic?: string; rpc: string[]; + watchRpc: string[]; writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; @@ -148,6 +149,7 @@ export type BotConfig = { key?: string; mnemonic?: string; rpc: string[]; + watchRpc: string[]; writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; From 03f6571d5f29ae2508593a0d0f217de678b4abf2 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 1 Nov 2024 01:10:12 +0000 Subject: [PATCH 18/32] Update sushiswap --- lib/sushiswap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sushiswap b/lib/sushiswap index 0f62029c..68913aeb 160000 --- a/lib/sushiswap +++ b/lib/sushiswap @@ -1 +1 @@ -Subproject commit 0f62029cda417f2395dde30a26bfc192dca64c6b +Subproject commit 68913aebb46a2d485a77391efcd0014fbdcf35da From 1f11326867d7468b19f935b809f18aead443dcb6 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Fri, 1 Nov 2024 23:56:52 +0000 Subject: [PATCH 19/32] fix test --- README.md | 7 +++---- lib/sushiswap | 2 +- src/cli.ts | 2 +- test/cli.test.js | 25 +++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4a18a02b..b2f87ab4 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ The app requires these arguments (all arguments can be set in env variables alte - `-k` or `--key`, Private key of wallet that performs the transactions, one of this or --mnemonic should be specified, requires `--wallet-count` and `--topup-amount`. Will override the 'BOT_WALLET_PRIVATEKEY' in env variables - `-m` or `--mnemonic`, Mnemonic phrase of wallet that performs the transactions, one of this or --key should be specified. Will override the 'MNEMONIC' in env variables - `-r` or `--rpc`, RPC URL(s) that will be provider for interacting with evm, use different providers if more than 1 is specified to prevent banning. Will override the 'RPC_URL' in env variables +- `--watch-rpc`, RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables - `--arb-address`, Address of the deployed arb contract, Will override the 'ARB_ADDRESS' in env variables - `--generic-arb-address`, Address of the deployed generic arb contract to perform inter-orderbook clears, Will override the 'GENERIC_ARB_ADDRESS' in env variables -- `--bot-min-balance` The minimum gas token balance the bot wallet must have. Will override the 'BOT_MIN_BALANCE' in env variables @@ -75,8 +76,7 @@ Other optional arguments are: - `--sleep`, Seconds to wait between each arb round, default is 10, Will override the 'SLEPP' in env variables - `--max-ratio`, Option to maximize maxIORatio, Will override the 'MAX_RATIO' in env variables - `--timeout`, Optional seconds to wait for the transaction to mine before disregarding it, Will override the 'TIMEOUT' in env variables -- `--write-rpc`, Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables" -- `--watch-rpc`, RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables" +- `--write-rpc`, Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables" - `--no-bundle`, Flag for not bundling orders based on pairs and clear each order individually. Will override the 'NO_BUNDLE' in env variables - `--hops`, Option to specify how many hops the binary search should do, default is 0 if left unspecified, Will override the 'HOPS' in env variables - `--retries`, Option to specify how many retries should be done for the same order, max value is 3, default is 1 if left unspecified, Will override the 'RETRIES' in env variables @@ -133,7 +133,6 @@ Other optional arguments are: `HyperBlast`, `KinetixV2`, `KinetixV3`, -`Camelot`, `Enosys`, `BlazeSwap`, @@ -157,7 +156,7 @@ MNEMONIC="" # for specifying more than 1 RPC in the env, separate them by a comma and a space RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.com/polygon/{API_KEY}" -# Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. +# Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. WRITE_RPC="" # RPC URLs to watch for new orders, should support required RPC methods diff --git a/lib/sushiswap b/lib/sushiswap index 68913aeb..ecab9c62 160000 --- a/lib/sushiswap +++ b/lib/sushiswap @@ -1 +1 @@ -Subproject commit 68913aebb46a2d485a77391efcd0014fbdcf35da +Subproject commit ecab9c62533e2f5287d7a152ccd3eb60f4e6f7e7 diff --git a/src/cli.ts b/src/cli.ts index d2394ab6..2c6849eb 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -133,7 +133,7 @@ const getOptions = async (argv: any, version?: string) => { ) .option( "--write-rpc ", - "Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables", + "Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables", ) .option( "--watch-rpc ", diff --git a/test/cli.test.js b/test/cli.test.js index 2c2117f4..06ea2aa4 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -99,6 +99,23 @@ describe("Test cli", async function () { try { await startup(["", "", "--key", `0x${"0".repeat(64)}`, "--rpc", "some-rpc"]); assert.fail("expected to fail, but resolved"); + } catch (error) { + const expected = "undefined watch RPC URL"; + assert.equal(error, expected); + } + + try { + await startup([ + "", + "", + "--key", + `0x${"0".repeat(64)}`, + "--rpc", + "some-rpc", + "--watch-rpc", + "some-rpc", + ]); + assert.fail("expected to fail, but resolved"); } catch (error) { const expected = "undefined arb contract address"; assert.equal(error, expected); @@ -112,6 +129,8 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", + "--watch-rpc", + "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -133,6 +152,8 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", + "--watch-rpc", + "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -155,6 +176,8 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", + "--watch-rpc", + "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -176,6 +199,8 @@ describe("Test cli", async function () { "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", "--rpc", "https://rpc.ankr.com/polygon", + "--watch-rpc", + "some-rpc", "--arb-address", `0x${"1".repeat(40)}`, "--orderbook-address", From ac2a9264c72407b6c77fc38b96404e713bde8a22 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 2 Nov 2024 02:09:40 +0000 Subject: [PATCH 20/32] update --- README.md | 9 ++------- example.env | 3 --- src/cli.ts | 8 +------- src/config.ts | 34 +++++++++++++++++++++++++--------- test/cli.test.js | 4 ++-- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 2db96e24..e72a38f5 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,7 @@ Other optional arguments are: - `--sleep`, Seconds to wait between each arb round, default is 10, Will override the 'SLEPP' in env variables - `--max-ratio`, Option to maximize maxIORatio, Will override the 'MAX_RATIO' in env variables - `--timeout`, Optional seconds to wait for the transaction to mine before disregarding it, Will override the 'TIMEOUT' in env variables -- `--write-rpc`, Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables" -- `--no-bundle`, Flag for not bundling orders based on pairs and clear each order individually. Will override the 'NO_BUNDLE' in env variables +- `--write-rpc`, Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables - `--hops`, Option to specify how many hops the binary search should do, default is 0 if left unspecified, Will override the 'HOPS' in env variables - `--retries`, Option to specify how many retries should be done for the same order, max value is 3, default is 1 if left unspecified, Will override the 'RETRIES' in env variables - `--pool-update-interval`, Option to specify time (in minutes) between pools updates, default is 15 minutes, Will override the 'POOL_UPDATE_INTERVAL' in env variables @@ -197,9 +196,6 @@ MAX_RATIO="true" # Optional seconds to wait for the transaction to mine before disregarding it TIMEOUT="" -# Flag for not bundling orders based on pairs and clear each order individually -NO_BUNDLE="false" - # number of hops of binary search, if left unspecified will be 7 by default HOPS=11 @@ -257,8 +253,7 @@ const RainArbBot = require("@rainprotocol/arb-bot"); const configOptions = { maxRatio : true, // option to maximize the maxIORatio flashbotRpc : "https://flashbot-rpc-url", // Optional Flashbot RPC URL - timeout : 300, // seconds to wait for tx to mine before disregarding it - bundle : true, // if orders should be bundled based on token pair or be handled individually + timeout : 300, // seconds to wait for tx to mine before disregarding it hops : 6, // The amount of hops of binary search retries : 1, // The amount of retries for the same order liquidityProviders : [ // list of liquidity providers to get quotes from (optional) diff --git a/example.env b/example.env index 65d28902..070c4052 100644 --- a/example.env +++ b/example.env @@ -51,9 +51,6 @@ MAX_RATIO="true" # Optional seconds to wait for the transaction to mine before disregarding it TIMEOUT="" -# Flag for not bundling orders based on pairs and clear each order individually -NO_BUNDLE="false" - # number of hops of binary search, if left unspecified will be 1 by default HOPS=11 diff --git a/src/cli.ts b/src/cli.ts index f89d3dcc..362fbc97 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,9 +26,9 @@ import { diag, trace, context, + DiagLogLevel, SpanStatusCode, DiagConsoleLogger, - DiagLogLevel, } from "@opentelemetry/api"; import { BasicTracerProvider, @@ -54,7 +54,6 @@ const ENV_OPTIONS = { orderOwner: process?.env?.ORDER_OWNER, sleep: process?.env?.SLEEP, maxRatio: process?.env?.MAX_RATIO?.toLowerCase() === "true" ? true : false, - bundle: process?.env?.NO_BUNDLE?.toLowerCase() === "true" ? false : true, publicRpc: process?.env?.PUBLIC_RPC?.toLowerCase() === "true" ? true : false, timeout: process?.env?.TIMEOUT, hops: process?.env?.HOPS, @@ -148,10 +147,6 @@ const getOptions = async (argv: any, version?: string) => { "--max-ratio", "Option to maximize maxIORatio, Will override the 'MAX_RATIO' in env variables", ) - .option( - "--no-bundle", - "Flag for not bundling orders based on pairs and clear each order individually. Will override the 'NO_BUNDLE' in env variables", - ) .option( "--hops ", "Option to specify how many hops the binary search should do, default is 1 if left unspecified, Will override the 'HOPS' in env variables", @@ -233,7 +228,6 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.botMinBalance = cmdOptions.botMinBalance || getEnv(ENV_OPTIONS.botMinBalance); cmdOptions.ownerProfile = cmdOptions.ownerProfile || getEnv(ENV_OPTIONS.ownerProfile); cmdOptions.route = cmdOptions.route || getEnv(ENV_OPTIONS.route); - cmdOptions.bundle = cmdOptions.bundle ? getEnv(ENV_OPTIONS.bundle) : false; cmdOptions.publicRpc = cmdOptions.publicRpc || getEnv(ENV_OPTIONS.publicRpc); if (cmdOptions.ownerProfile) { const profiles: Record = {}; diff --git a/src/config.ts b/src/config.ts index 148331d8..a01e3956 100644 --- a/src/config.ts +++ b/src/config.ts @@ -5,15 +5,15 @@ import { ChainId, ChainKey } from "sushi/chain"; import { DataFetcher, LiquidityProviders } from "sushi/router"; import { BotConfig, BotDataFetcher, ChainConfig, ViemClient } from "./types"; import { - createWalletClient, + http, fallback, HDAccount, - http, - PrivateKeyAccount, - publicActions, + webSocket, PublicClient, + publicActions, walletActions, - webSocket, + PrivateKeyAccount, + createWalletClient, } from "viem"; import { STABLES, @@ -75,15 +75,31 @@ export async function createViemClient( const urls = rpcs?.filter((v) => typeof v === "string") ?? []; const topRpcs = urls.map((v) => v.startsWith("http") - ? http(v, { timeout, onFetchRequest: config?.onFetchRequest, onFetchResponse: config?.onFetchResponse }) - : webSocket(v, { timeout, keepAlive: true, reconnect: true, onFetchRequest: config?.onFetchRequest, onFetchResponse: config?.onFetchResponse }), + ? http(v, { + timeout, + onFetchRequest: config?.onFetchRequest, + onFetchResponse: config?.onFetchResponse, + }) + : webSocket(v, { + timeout, + keepAlive: true, + reconnect: true, + }), ); const fallbacks = (fallbackRpcs[chainId] ?? []) .filter((v) => !urls.includes(v)) .map((v) => v.startsWith("http") - ? http(v, { timeout, onFetchRequest: config?.onFetchRequest, onFetchResponse: config?.onFetchResponse }) - : webSocket(v, { timeout, keepAlive: true, reconnect: true, onFetchRequest: config?.onFetchRequest, onFetchResponse: config?.onFetchResponse }), + ? http(v, { + timeout, + onFetchRequest: config?.onFetchRequest, + onFetchResponse: config?.onFetchResponse, + }) + : webSocket(v, { + timeout, + keepAlive: true, + reconnect: true, + }), ); const transport = !topRpcs.length ? fallback(fallbacks, configuration) diff --git a/test/cli.test.js b/test/cli.test.js index 24a22006..5395fe55 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -219,8 +219,8 @@ describe("Test cli", async function () { route: "single", rpcRecords: { "https://rpc.ankr.com/polygon": { - req: 1, - success: 1, + req: 2, + success: 2, failure: 0, }, }, From f02da0166be1cc29e3b588b9d340a31b40e8df80 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 2 Nov 2024 03:37:13 +0000 Subject: [PATCH 21/32] update --- src/processOrders.ts | 10 +++++----- src/watcher.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index 8d917954..456b97da 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -211,7 +211,7 @@ export const processOrders = async ( viemClient, dataFetcher, signer, - flashbotSigner: writeSigner, + writeSigner, arb, genericArb, orderbook, @@ -351,7 +351,7 @@ export async function processPair(args: { viemClient: PublicClient; dataFetcher: BotDataFetcher; signer: ViemClient; - flashbotSigner: ViemClient | undefined; + writeSigner: ViemClient | undefined; arb: Contract; genericArb: Contract | undefined; orderbook: Contract; @@ -364,7 +364,7 @@ export async function processPair(args: { viemClient, dataFetcher, signer, - flashbotSigner, + writeSigner, arb, genericArb, orderbook, @@ -602,8 +602,8 @@ export async function processPair(args: { let txhash, txUrl; try { txhash = - flashbotSigner !== undefined - ? await flashbotSigner.sendTransaction(rawtx) + writeSigner !== undefined + ? await writeSigner.sendTransaction(rawtx) : await signer.sendTransaction(rawtx); txUrl = config.chain.blockExplorers?.default.url + "/tx/" + txhash; diff --git a/src/watcher.ts b/src/watcher.ts index e99f7aac..8e87228f 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -209,7 +209,7 @@ export async function handleNewOrderLogs( ); span?.setAttribute( `orderbooksChanges.${orderbook}.removedOrders`, - orderLogs.filter((v) => v.type === "add").map((v) => v.order.orderHash), + orderLogs.filter((v) => v.type === "remove").map((v) => v.order.orderHash), ); } for (let i = 0; i < orderLogs.length; i++) { From d0863a506c337c3f54997cda7e196269b1c993e7 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 11 Nov 2024 01:58:37 +0000 Subject: [PATCH 22/32] rm rpc watcher, use sg instead --- src/cli.ts | 105 +++++++------- src/order.ts | 118 ++++++++++++++-- src/query.ts | 172 ++++++++++++++++++++++- src/utils.ts | 2 +- src/watcher.ts | 276 ------------------------------------ test/cli.test.js | 4 +- test/watcher.test.js | 323 ------------------------------------------- 7 files changed, 339 insertions(+), 661 deletions(-) delete mode 100644 src/watcher.ts delete mode 100644 test/watcher.test.js diff --git a/src/cli.ts b/src/cli.ts index f46b1995..aecb3fc6 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,27 +1,26 @@ import { config } from "dotenv"; -import { SgOrder } from "./query"; import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; +import { getAddOrders, SgOrder } from "./query"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; +import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; -import { sleep, getOrdersTokens, isBigNumberish, getblockNumber } from "./utils"; -import { getOrderbookOwnersProfileMapFromSg, prepareOrdersForRound } from "./order"; import { manageAccounts, rotateProviders, sweepToMainWallet, sweepToEth } from "./account"; import { - watchOrderbook, - watchAllOrderbooks, - WatchedOrderbookOrders, - handleOrderbooksNewLogs, -} from "./watcher"; + prepareOrdersForRound, + getOrderbookOwnersProfileMapFromSg, + handleAddOrderbookOwnersProfileMap, + handleRemoveOrderbookOwnersProfileMap, +} from "./order"; import { diag, trace, @@ -413,6 +412,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? } } } + const lastReadOrdersTimestamp = Math.floor(Date.now() / 1000); const tokens = getOrdersTokens(ordersDetails); options.tokens = [...tokens]; @@ -425,7 +425,6 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? tracer, ctx, ); - const blockNumber = (await getblockNumber(config.viemClient as any as ViemClient)) ?? 1n; return { roundGap, @@ -439,7 +438,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? (options as CliOptions).ownerProfile, ), tokens, - blockNumber, + lastReadOrdersTimestamp, }; } @@ -485,7 +484,7 @@ export const main = async (argv: any, version?: string) => { config, orderbooksOwnersProfileMap, tokens, - blockNumber: bn, + lastReadOrdersTimestamp, } = await tracer.startActiveSpan("startup", async (startupSpan) => { const ctx = trace.setSpan(context.active(), startupSpan); try { @@ -508,14 +507,11 @@ export const main = async (argv: any, version?: string) => { } }); - let blockNumber = bn; - const obs: string[] = []; - const watchedOrderbooksOrders: Record = {}; - orderbooksOwnersProfileMap.forEach((_, ob) => { - obs.push(ob.toLowerCase()); - }); - const unwatchers = watchAllOrderbooks(obs, config.watchClient, watchedOrderbooksOrders); - + const lastReadOrdersTimestampMap = options.subgraph.map((v) => ({ + sg: v, + lastReadTimestampAdd: lastReadOrdersTimestamp, + lastReadTimestampRemove: lastReadOrdersTimestamp, + })); const day = 24 * 60 * 60 * 1000; let lastGasReset = Date.now() + day; let lastInterval = Date.now() + poolUpdateInterval; @@ -539,27 +535,6 @@ export const main = async (argv: any, version?: string) => { "meta.dockerTag": process?.env?.DOCKER_TAG ?? "N/A", }); - // watch new obs - for (const newOb of newMeta["meta.orderbooks"]) { - const ob = newOb.toLowerCase(); - if (!obs.includes(ob)) { - obs.push(ob); - if (!watchedOrderbooksOrders[ob]) { - watchedOrderbooksOrders[ob] = { orderLogs: [] }; - } - if (!unwatchers[ob]) { - unwatchers[ob] = watchOrderbook( - ob, - config.watchClient, - watchedOrderbooksOrders[ob], - blockNumber, - ); - } - } - } - const tempBn = await getblockNumber(config.viemClient as any as ViemClient); - if (tempBn !== undefined) blockNumber = (tempBn * 95n) / 100n; - await tracer.startActiveSpan( "check-wallet-balance", {}, @@ -735,15 +710,49 @@ export const main = async (argv: any, version?: string) => { } try { - // check for new orders - await handleOrderbooksNewLogs( - orderbooksOwnersProfileMap, - watchedOrderbooksOrders, - config.viemClient as any as ViemClient, - tokens, - options.ownerProfile, - roundSpan, + // check for new orders changes + const now = Math.floor(Date.now() / 1000); + const addOrdersResult = await Promise.allSettled( + lastReadOrdersTimestampMap.map((v) => + getAddOrders(v.sg, v.lastReadTimestampAdd, now, options.timeout, roundSpan), + ), ); + for (let i = 0; i < addOrdersResult.length; i++) { + const res = addOrdersResult[i]; + if (res.status === "fulfilled") { + lastReadOrdersTimestampMap[i].lastReadTimestampAdd = now; + await handleAddOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap, + res.value.map((v) => v.order), + config.viemClient as any as ViemClient, + tokens, + options.ownerProfile, + roundSpan, + ); + } + } + const rmOrdersResult = await Promise.allSettled( + lastReadOrdersTimestampMap.map((v) => + getAddOrders( + v.sg, + v.lastReadTimestampRemove, + now, + options.timeout, + roundSpan, + ), + ), + ); + for (let i = 0; i < rmOrdersResult.length; i++) { + const res = rmOrdersResult[i]; + if (res.status === "fulfilled") { + lastReadOrdersTimestampMap[i].lastReadTimestampRemove = now; + await handleRemoveOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap, + res.value.map((v) => v.order), + roundSpan, + ); + } + } } catch { /**/ } diff --git a/src/order.ts b/src/order.ts index 67cde083..bf9af1c0 100644 --- a/src/order.ts +++ b/src/order.ts @@ -1,6 +1,7 @@ import { OrderV3 } from "./abis"; import { SgOrder } from "./query"; -import { toOrder } from "./watcher"; +import { Span } from "@opentelemetry/api"; +import { hexlify } from "ethers/lib/utils"; import { getTokenSymbol, shuffleArray } from "./utils"; import { decodeAbiParameters, parseAbiParameters } from "viem"; import { @@ -19,6 +20,28 @@ import { */ export const DEFAULT_OWNER_LIMIT = 25 as const; +export function toOrder(orderLog: any): Order { + return { + owner: orderLog.owner.toLowerCase(), + nonce: orderLog.nonce.toLowerCase(), + evaluable: { + interpreter: orderLog.evaluable.interpreter.toLowerCase(), + store: orderLog.evaluable.store.toLowerCase(), + bytecode: orderLog.evaluable.bytecode.toLowerCase(), + }, + validInputs: orderLog.validInputs.map((v: any) => ({ + token: v.token.toLowerCase(), + decimals: v.decimals, + vaultId: hexlify(v.vaultId), + })), + validOutputs: orderLog.validOutputs.map((v: any) => ({ + token: v.token.toLowerCase(), + decimals: v.decimals, + vaultId: hexlify(v.vaultId), + })), + }; +} + /** * Get all pairs of an order */ @@ -96,17 +119,19 @@ export async function getOrderPairs( } return pairs; } + /** - * Get a map of per owner orders per orderbook - * @param ordersDetails - Order details queried from subgraph + * Handles new orders fetched from sg to the owner profile map */ -export async function getOrderbookOwnersProfileMapFromSg( +export async function handleAddOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, ordersDetails: SgOrder[], viemClient: ViemClient, tokens: TokenDetails[], ownerLimits?: Record, -): Promise { - const orderbookOwnersProfileMap: OrderbooksOwnersProfileMap = new Map(); + span?: Span, +) { + const changes: Record = {}; for (let i = 0; i < ordersDetails.length; i++) { const orderDetails = ordersDetails[i]; const orderbook = orderDetails.orderbook.id.toLowerCase(); @@ -116,11 +141,18 @@ export async function getOrderbookOwnersProfileMapFromSg( orderDetails.orderBytes as `0x${string}`, )[0], ); - const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); + if (span) { + if (!changes[orderbook]) changes[orderbook] = []; + if (!changes[orderbook].includes(orderDetails.orderHash.toLowerCase())) { + changes[orderbook].push(orderDetails.orderHash.toLowerCase()); + } + } + const orderbookOwnerProfileItem = orderbooksOwnersProfileMap.get(orderbook); if (orderbookOwnerProfileItem) { const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); if (ownerProfile) { - if (!ownerProfile.orders.has(orderDetails.orderHash.toLowerCase())) { + const order = ownerProfile.orders.get(orderDetails.orderHash.toLowerCase()); + if (!order) { ownerProfile.orders.set(orderDetails.orderHash.toLowerCase(), { active: true, order: orderStruct, @@ -132,6 +164,8 @@ export async function getOrderbookOwnersProfileMapFromSg( ), consumedTakeOrders: [], }); + } else { + if (!order.active) order.active = true; } } else { const ordersProfileMap: OrdersProfileMap = new Map(); @@ -159,10 +193,74 @@ export async function getOrderbookOwnersProfileMapFromSg( limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, orders: ordersProfileMap, }); - orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); + orderbooksOwnersProfileMap.set(orderbook, ownerProfileMap); + } + } + if (span) { + for (const orderbook in changes) { + span.setAttribute(`orderbooksChanges.${orderbook}.addedOrders`, changes[orderbook]); + } + } +} + +/** + * Handles new removed orders fetched from sg to the owner profile map + */ +export async function handleRemoveOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, + ordersDetails: SgOrder[], + span?: Span, +) { + const changes: Record = {}; + for (let i = 0; i < ordersDetails.length; i++) { + const orderDetails = ordersDetails[i]; + const orderbook = orderDetails.orderbook.id.toLowerCase(); + const orderStruct = toOrder( + decodeAbiParameters( + parseAbiParameters(OrderV3), + orderDetails.orderBytes as `0x${string}`, + )[0], + ); + if (span) { + if (!changes[orderbook]) changes[orderbook] = []; + if (!changes[orderbook].includes(orderDetails.orderHash.toLowerCase())) { + changes[orderbook].push(orderDetails.orderHash.toLowerCase()); + } + } + const orderbookOwnerProfileItem = orderbooksOwnersProfileMap.get(orderbook); + if (orderbookOwnerProfileItem) { + const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); + if (ownerProfile) { + ownerProfile.orders.delete(orderDetails.orderHash.toLowerCase()); + } + } + } + if (span) { + for (const orderbook in changes) { + span.setAttribute(`orderbooksChanges.${orderbook}.removedOrders`, changes[orderbook]); } } - return orderbookOwnersProfileMap; +} + +/** + * Get a map of per owner orders per orderbook + * @param ordersDetails - Order details queried from subgraph + */ +export async function getOrderbookOwnersProfileMapFromSg( + ordersDetails: SgOrder[], + viemClient: ViemClient, + tokens: TokenDetails[], + ownerLimits?: Record, +): Promise { + const orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap = new Map(); + await handleAddOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap, + ordersDetails, + viemClient, + tokens, + ownerLimits, + ); + return orderbooksOwnersProfileMap; } /** diff --git a/src/query.ts b/src/query.ts index c2c18d03..5973de7e 100644 --- a/src/query.ts +++ b/src/query.ts @@ -1,4 +1,6 @@ import axios from "axios"; +import { errorSnapshot } from "./error"; +import { Span } from "@opentelemetry/api"; export type SgOrder = { id: string; @@ -30,6 +32,11 @@ export type SgOrder = { }[]; }; +export type NewSgOrder = { + order: SgOrder; + timestamp: number; +}; + /** * Method to get the subgraph query body with optional filters * @param orderHash - The order hash to apply as filter @@ -47,7 +54,7 @@ export function getQueryPaginated( const orderHashFilter = orderHash ? `, orderHash: "${orderHash.toLowerCase()}"` : ""; const orderbookFilter = orderbook ? `, orderbook: "${orderbook.toLowerCase()}"` : ""; return `{ - orders(first: 100, skip: ${skip}, where: {active: true${orderbookFilter}${orderHashFilter}${ownerFilter}}) { + orders(first: 100, skip: ${skip}, orderBy: timestampAdded, orderDirection: desc, where: {active: true${orderbookFilter}${orderHashFilter}${ownerFilter}}) { id owner orderHash @@ -133,3 +140,166 @@ export const statusCheckQuery = `{ } } }`; + +export const getRemoveOrdersQuery = (start: number, end: number) => { + return `removeOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { + order { + id + owner + orderHash + orderBytes + active + nonce + orderbook { + id + } + inputs { + balance + vaultId + token { + address + decimals + symbol + } + } + outputs { + balance + vaultId + token { + address + decimals + symbol + } + } + } + transaction { + timestamp + } +}`; +}; + +export const getAddOrdersQuery = (start: number, end: number) => { + return `addOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_gt: "${end.toString()}" } }) { + order { + id + owner + orderHash + orderBytes + active + nonce + orderbook { + id + } + inputs { + balance + vaultId + token { + address + decimals + symbol + } + } + outputs { + balance + vaultId + token { + address + decimals + symbol + } + } + } + transaction { + timestamp + } +}`; +}; + +/** + * Fecthes the remove orders from the given subgraph in the given timeframe + * @param subgraph - The subgraph url + * @param startTimestamp - start timestamp range + * @param endTimestamp - end timestamp range + * @param timeout - promise timeout + */ +export async function getRemoveOrders( + subgraph: string, + startTimestamp: number, + endTimestamp: number, + timeout?: number, + span?: Span, +) { + const removeOrders: NewSgOrder[] = []; + try { + const res = await axios.post( + subgraph, + { query: getRemoveOrdersQuery(startTimestamp, endTimestamp) }, + { headers: { "Content-Type": "application/json" }, timeout }, + ); + if (typeof res?.data?.data?.removeOrders !== "undefined") { + res.data.data.removeOrders.forEach((v: any) => { + if (typeof v?.order?.active === "boolean" && !v.order.active) { + if (!removeOrders.find((e) => e.order.id === v.order.id)) { + removeOrders.push({ + order: v.order as SgOrder, + timestamp: Number(v.transaction.timestamp), + }); + } + } + }); + } else { + span?.addEvent(`Failed to get new removed orders ${subgraph}: invalid response`); + throw "invalid response"; + } + } catch (error) { + span?.addEvent(errorSnapshot(`Failed to get new removed orders ${subgraph}`, error)); + } + + removeOrders.sort((a, b) => b.timestamp - a.timestamp); + return removeOrders; +} + +/** + * Fecthes the add orders from the given subgraph in the given timeframe + * @param subgraph - The subgraph url + * @param startTimestamp - start timestamp range + * @param endTimestamp - end timestamp range + * @param timeout - promise timeout + */ +export async function getAddOrders( + subgraph: string, + startTimestamp: number, + endTimestamp: number, + timeout?: number, + span?: Span, +) { + const addOrders: NewSgOrder[] = []; + try { + const res = await axios.post( + subgraph, + { query: getAddOrdersQuery(startTimestamp, endTimestamp) }, + { headers: { "Content-Type": "application/json" }, timeout }, + ); + if (typeof res?.data?.data?.addOrders !== "undefined") { + res.data.data.addOrders.forEach((v: any) => { + if (typeof v?.order?.active === "boolean" && v.order.active) { + if (!addOrders.find((e) => e.order.id === v.order.id)) { + addOrders.push({ + order: v.order as SgOrder, + timestamp: Number(v.transaction.timestamp), + }); + } + } + }); + } else { + span?.addEvent(`Failed to get new orders ${subgraph}: invalid response`); + throw "invalid response"; + } + } catch (error) { + span?.addEvent(errorSnapshot(`Failed to get new orders from subgraph ${subgraph}`, error)); + throw error; + } + + addOrders.sort((a, b) => b.timestamp - a.timestamp); + return addOrders; +} diff --git a/src/utils.ts b/src/utils.ts index d6743b33..537a4920 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1325,7 +1325,7 @@ export function isBigNumberish(value: any): value is BigNumberish { /** * Get block number with retries, using viem client */ -export async function getblockNumber(viemClient: ViemClient): Promise { +export async function getBlockNumber(viemClient: ViemClient): Promise { for (let i = 0; i < 3; i++) { try { return await viemClient.getBlockNumber(); diff --git a/src/watcher.ts b/src/watcher.ts deleted file mode 100644 index 8e87228f..00000000 --- a/src/watcher.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { Span } from "@opentelemetry/api"; -import { hexlify } from "ethers/lib/utils"; -import { orderbookAbi as abi } from "./abis"; -import { DEFAULT_OWNER_LIMIT, getOrderPairs } from "./order"; -import { parseAbi, WatchContractEventReturnType } from "viem"; -import { - Order, - ViemClient, - TokenDetails, - OrdersProfileMap, - OwnersProfileMap, - OrderbooksOwnersProfileMap, -} from "./types"; -import { errorSnapshot } from "./error"; - -type OrderEventLog = { - sender: `0x${string}`; - orderHash: `0x${string}`; - order: { - owner: `0x${string}`; - evaluable: { - interpreter: `0x${string}`; - store: `0x${string}`; - bytecode: `0x${string}`; - }; - validInputs: readonly { - token: `0x${string}`; - decimals: number; - vaultId: bigint; - }[]; - validOutputs: readonly { - token: `0x${string}`; - decimals: number; - vaultId: bigint; - }[]; - nonce: `0x${string}`; - }; -}; -export type OrderArgsLog = { - sender: `0x${string}`; - orderHash: `0x${string}`; - order: Order; -}; -export type OrderLog = { - type: "add" | "remove"; - order: OrderArgsLog; - block: number; - logIndex: number; - txHash: string; -}; -export type WatchedOrderbookOrders = { orderLogs: OrderLog[] }; - -function logToOrder(orderLog: OrderEventLog): OrderArgsLog { - return { - sender: orderLog.sender.toLowerCase() as `0x${string}`, - orderHash: orderLog.orderHash.toLowerCase() as `0x${string}`, - order: toOrder(orderLog.order), - }; -} - -export function toOrder(orderLog: any): Order { - return { - owner: orderLog.owner.toLowerCase(), - nonce: orderLog.nonce.toLowerCase(), - evaluable: { - interpreter: orderLog.evaluable.interpreter.toLowerCase(), - store: orderLog.evaluable.store.toLowerCase(), - bytecode: orderLog.evaluable.bytecode.toLowerCase(), - }, - validInputs: orderLog.validInputs.map((v: any) => ({ - token: v.token.toLowerCase(), - decimals: v.decimals, - vaultId: hexlify(v.vaultId), - })), - validOutputs: orderLog.validOutputs.map((v: any) => ({ - token: v.token.toLowerCase(), - decimals: v.decimals, - vaultId: hexlify(v.vaultId), - })), - }; -} - -export const orderbookAbi = parseAbi([abi[0], abi[1]]); - -/** - * Applies an event watcher for a specified orderbook - */ -export function watchOrderbook( - orderbook: string, - viemClient: ViemClient, - watchedOrderbookOrders: WatchedOrderbookOrders, - fromBlock?: bigint, -): WatchContractEventReturnType { - return viemClient.watchContractEvent({ - address: orderbook as `0x${string}`, - abi: orderbookAbi, - pollingInterval: 30_000, - fromBlock, - onLogs: (logs) => { - logs.forEach((log) => { - if (log) { - try { - watchedOrderbookOrders.orderLogs.push({ - type: log.eventName === "AddOrderV2" ? "add" : "remove", - logIndex: log.logIndex, - block: Number(log.blockNumber), - txHash: log.transactionHash, - order: logToOrder(log.args as any as OrderEventLog), - }); - } catch (error) { - // eslint-disable-next-line no-console - console.warn( - errorSnapshot( - `Failed to handle orderbook ${orderbook} new event log`, - error, - ), - ); - // eslint-disable-next-line no-console - console.log("\nOriginal log:\n", log); - } - } - }); - }, - onError: (error) => - // eslint-disable-next-line no-console - console.warn( - errorSnapshot( - `An error occured during watching new events logs of orderbook ${orderbook}`, - error, - ), - ), - }); -} - -/** - * Applies event watcher all known orderbooks - * @returns Unwatchers for all orderbooks - */ -export function watchAllOrderbooks( - orderbooks: string[], - viemClient: ViemClient, - watchedOrderbooksOrders: Record, -): Record { - const allUnwatchers: Record = {}; - for (const v of orderbooks) { - const ob = v.toLowerCase(); - if (!watchedOrderbooksOrders[ob]) { - watchedOrderbooksOrders[ob] = { orderLogs: [] }; - } - allUnwatchers[ob] = watchOrderbook(ob, viemClient, watchedOrderbooksOrders[ob]); - } - return allUnwatchers; -} - -/** - * Unwatches all orderbooks event watchers - */ -export function unwatchAllOrderbooks(unwatchers: Record) { - for (const ob in unwatchers) { - unwatchers[ob]?.(); - } -} - -/** - * Hanldes all new order logs of all watched orderbooks - */ -export async function handleOrderbooksNewLogs( - orderbooksOwnersProfileMap: OrderbooksOwnersProfileMap, - watchedOrderbooksOrders: Record, - viemClient: ViemClient, - tokens: TokenDetails[], - ownerLimits?: Record, - span?: Span, -) { - for (const ob in watchedOrderbooksOrders) { - const watchedOrderbookLogs = watchedOrderbooksOrders[ob]; - const logs = watchedOrderbookLogs.orderLogs.splice(0); - // make sure logs are sorted before applying them to the map - logs.sort((a, b) => a.block - b.block || a.logIndex - b.logIndex); - await handleNewOrderLogs( - ob, - logs, - orderbooksOwnersProfileMap, - viemClient, - tokens, - ownerLimits, - span, - ); - } -} - -/** - * Handles new order logs for an orderbook - */ -export async function handleNewOrderLogs( - orderbook: string, - orderLogs: OrderLog[], - orderbookOwnersProfileMap: OrderbooksOwnersProfileMap, - viemClient: ViemClient, - tokens: TokenDetails[], - ownerLimits?: Record, - span?: Span, -) { - orderbook = orderbook.toLowerCase(); - if (orderLogs.length) { - span?.setAttribute( - `orderbooksChanges.${orderbook}.addedOrders`, - orderLogs.filter((v) => v.type === "add").map((v) => v.order.orderHash), - ); - span?.setAttribute( - `orderbooksChanges.${orderbook}.removedOrders`, - orderLogs.filter((v) => v.type === "remove").map((v) => v.order.orderHash), - ); - } - for (let i = 0; i < orderLogs.length; i++) { - const orderLog = orderLogs[i].order; - const orderStruct = orderLog.order; - const orderbookOwnerProfileItem = orderbookOwnersProfileMap.get(orderbook); - if (orderLogs[i].type === "add") { - if (orderbookOwnerProfileItem) { - const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); - if (ownerProfile) { - const order = ownerProfile.orders.get(orderLog.orderHash.toLowerCase()); - if (!order) { - ownerProfile.orders.set(orderLog.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), - consumedTakeOrders: [], - }); - } else { - order.active = true; - } - } else { - const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderLog.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), - consumedTakeOrders: [], - }); - orderbookOwnerProfileItem.set(orderStruct.owner.toLowerCase(), { - limit: - ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, - orders: ordersProfileMap, - }); - } - } else { - const ordersProfileMap: OrdersProfileMap = new Map(); - ordersProfileMap.set(orderLog.orderHash.toLowerCase(), { - active: true, - order: orderStruct, - takeOrders: await getOrderPairs(orderStruct, viemClient, tokens), - consumedTakeOrders: [], - }); - const ownerProfileMap: OwnersProfileMap = new Map(); - ownerProfileMap.set(orderStruct.owner.toLowerCase(), { - limit: ownerLimits?.[orderStruct.owner.toLowerCase()] ?? DEFAULT_OWNER_LIMIT, - orders: ordersProfileMap, - }); - orderbookOwnersProfileMap.set(orderbook, ownerProfileMap); - } - } else { - if (orderbookOwnerProfileItem) { - const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); - if (ownerProfile) { - const order = ownerProfile.orders.get(orderLog.orderHash.toLowerCase()); - if (order) { - order.active = false; - order.takeOrders.push(...order.consumedTakeOrders.splice(0)); - } - } - } - } - } -} diff --git a/test/cli.test.js b/test/cli.test.js index 304bada4..45534a6f 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -220,8 +220,8 @@ describe("Test cli", async function () { route: "single", rpcRecords: { "https://rpc.ankr.com/polygon/": { - req: 2, - success: 2, + req: 1, + success: 1, failure: 0, cache: {}, }, diff --git a/test/watcher.test.js b/test/watcher.test.js deleted file mode 100644 index b4a3d400..00000000 --- a/test/watcher.test.js +++ /dev/null @@ -1,323 +0,0 @@ -const { assert } = require("chai"); -const { ethers } = require("hardhat"); -const { OrderV3 } = require("../src/abis"); -const mockServer = require("mockttp").getLocal(); -const { handleOrderbooksNewLogs } = require("../src/watcher"); -const { getOrderbookOwnersProfileMapFromSg } = require("../src/order"); -const { - utils: { hexlify, randomBytes }, -} = require("ethers"); - -describe("Test watchers", async function () { - beforeEach(() => mockServer.start(8899)); - afterEach(() => mockServer.stop()); - - const tokens = []; - function getOrderStruct(order) { - return { - nonce: order.nonce, - owner: order.owner.toLowerCase(), - evaluable: { - interpreter: `0x${"1".repeat(40)}`, - store: `0x${"2".repeat(40)}`, - bytecode: "0x1234", - }, - validInputs: order.inputs.map((v) => ({ - token: v.token.address.toLowerCase(), - decimals: v.token.decimals, - vaultId: v.vaultId, - })), - validOutputs: order.outputs.map((v) => ({ - token: v.token.address.toLowerCase(), - decimals: v.token.decimals, - vaultId: v.vaultId, - })), - }; - } - const getOrderbookOwnersProfileMap = async () => { - const order1 = { - id: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", - orderHash: "0x004349d76523bce3b6aeec93cf4c2a396b9cb71bc07f214e271cab363a0c89eb", - owner: "0x0f47a0c7f86a615606ca315ad83c3e302b474bd6", - orderBytes: "", - active: true, - nonce: `0x${"0".repeat(64)}`, - orderbook: { - id: `0x${"2".repeat(40)}`, - }, - inputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", - }, - }, - ], - outputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", - }, - }, - ], - }; - const orderStruct1 = getOrderStruct(order1); - const orderBytes1 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct1]); - order1.struct = orderStruct1; - order1.orderBytes = orderBytes1; - - const order2 = { - id: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", - orderHash: "0x008817a4b6f264326ef14357df54e48b9c064051f54f3877807970bb98096c01", - owner: "0x0eb840e5acd0125853ad630663d3a62e673c22e6", - orderBytes: "", - active: true, - nonce: `0x${"0".repeat(64)}`, - orderbook: { - id: `0x${"2".repeat(40)}`, - }, - inputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", - }, - }, - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", - }, - }, - ], - outputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: "0xc2132d05d31c914a87c6611c10748aeb04b58e8f", - decimals: 6, - symbol: "USDT", - }, - }, - { - balance: "1", - vaultId: "1", - token: { - address: "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270", - decimals: 18, - symbol: "WMATIC", - }, - }, - ], - }; - const orderStruct2 = getOrderStruct(order2); - const orderBytes2 = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct2]); - order2.struct = orderStruct2; - order2.orderBytes = orderBytes2; - - return [ - await getOrderbookOwnersProfileMapFromSg([order1, order2], undefined, tokens, {}), - order1, - order2, - ]; - }; - - const getNewOrder = (orderbook, owner) => { - const orderHash = hexlify(randomBytes(32)); - const order = { - id: orderHash, - orderHash: orderHash, - owner, - orderBytes: "", - active: true, - nonce: `0x${"0".repeat(64)}`, - orderbook: { - id: orderbook, - }, - inputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: hexlify(randomBytes(20)), - decimals: 6, - symbol: "NewToken1", - }, - }, - ], - outputs: [ - { - balance: "1", - vaultId: "1", - token: { - address: hexlify(randomBytes(20)), - decimals: 18, - symbol: "NewToken2", - }, - }, - ], - }; - const orderStruct = getOrderStruct(order); - const orderBytes = ethers.utils.defaultAbiCoder.encode([OrderV3], [orderStruct]); - order.orderBytes = orderBytes; - order.struct = orderStruct; - return order; - }; - - it("should handle orderbooks new logs into orderbook owner map, add and remove", async function () { - const [orderbooksOwnersProfileMap, order1] = await getOrderbookOwnersProfileMap(); - - const newOrderbook = hexlify(randomBytes(20)); - const newOwner1 = hexlify(randomBytes(20)); - const newOrder1 = getNewOrder(newOrderbook, newOwner1); - - const newOwner2 = hexlify(randomBytes(20)); - const newOrder2 = getNewOrder(`0x${"2".repeat(40)}`, newOwner2); - - const newOrderbookLogs = { - [`0x${"2".repeat(40)}`]: { - orderLogs: [ - { - type: "remove", - block: 1, - logIndex: 1, - order: { - sender: order1.owner, - orderHash: order1.orderHash, - order: order1.struct, - }, - }, - { - type: "add", - block: 2, - logIndex: 1, - order: { - sender: newOwner2, - orderHash: newOrder2.orderHash, - order: newOrder2.struct, - }, - }, - ], - }, - [newOrderbook]: { - orderLogs: [ - { - type: "add", - block: 2, - logIndex: 1, - order: { - sender: newOwner1, - orderHash: newOrder1.orderHash, - order: newOrder1.struct, - }, - }, - ], - }, - }; - await handleOrderbooksNewLogs( - orderbooksOwnersProfileMap, - newOrderbookLogs, - undefined, - tokens, - {}, - ); - - const expectedMap = (await getOrderbookOwnersProfileMap())[0]; - expectedMap - .get(`0x${"2".repeat(40)}`) - .get(order1.owner.toLowerCase()) - .orders.get(order1.orderHash.toLowerCase()).active = false; - expectedMap.get(`0x${"2".repeat(40)}`).set(newOwner2, { - limits: 25, - orders: new Map([ - [ - newOrder2.orderHash.toLowerCase(), - { - active: true, - order: newOrder2, - consumedTakeOrders: [], - takeOrders: [ - { - buyToken: newOrder2.struct.validInputs[0].token, - buyTokenSymbol: newOrder2.inputs[0].token.symbol, - buyTokenDecimals: newOrder2.struct.validInputs[0].decimals, - sellToken: newOrder2.struct.validOutputs[0].token, - sellTokenSymbol: newOrder2.outputs[0].token.symbol, - sellTokenDecimals: newOrder2.struct.validOutputs[0].decimals, - takeOrder: { - order: newOrder2.struct, - inputIOIndex: 0, - outputIOIndex: 0, - signedContext: [], - }, - }, - ], - }, - ], - ]), - }); - expectedMap.set( - newOrderbook, - new Map([ - [ - newOwner1, - { - limits: 25, - orders: new Map([ - [ - newOrder1.orderHash.toLowerCase(), - { - active: true, - order: newOrder1, - consumedTakeOrders: [], - takeOrders: [ - { - buyToken: newOrder1.struct.validInputs[0].token, - buyTokenSymbol: newOrder1.inputs[0].token.symbol, - buyTokenDecimals: - newOrder1.struct.validInputs[0].decimals, - sellToken: newOrder1.struct.validOutputs[0].token, - sellTokenSymbol: newOrder1.outputs[0].token.symbol, - sellTokenDecimals: - newOrder1.struct.validOutputs[0].decimals, - takeOrder: { - order: newOrder1.struct, - inputIOIndex: 0, - outputIOIndex: 0, - signedContext: [], - }, - }, - ], - }, - ], - ]), - }, - ], - ]), - ); - - const result = Array.from(orderbooksOwnersProfileMap).map((v) => [ - v[0], - Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), - ]); - const expected = Array.from(expectedMap).map((v) => [ - v[0], - Array.from(v[1]).map((e) => [e[0], Array.from(e[1])]), - ]); - assert.deepEqual(result, expected); - }); -}); From cafaf5ddcfdb493903529bf4cbed8a78e525e9d0 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 11 Nov 2024 02:14:17 +0000 Subject: [PATCH 23/32] update --- README.md | 4 ---- example.env | 3 --- src/cli.ts | 13 +------------ src/index.ts | 11 ----------- src/types.ts | 3 --- test/cli.test.js | 25 ------------------------- test/orders.test.js | 2 +- 7 files changed, 2 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 826ff31b..0801a0d5 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,6 @@ The app requires these arguments (all arguments can be set in env variables alte - `-k` or `--key`, Private key of wallet that performs the transactions, one of this or --mnemonic should be specified. Will override the 'BOT_WALLET_PRIVATEKEY' in env variables - `-m` or `--mnemonic`, Mnemonic phrase of wallet that performs the transactions, one of this or --key should be specified, requires `--wallet-count` and `--topup-amount`. Will override the 'MNEMONIC' in env variables - `-r` or `--rpc`, RPC URL(s) that will be provider for interacting with evm, use different providers if more than 1 is specified to prevent banning. Will override the 'RPC_URL' in env variables -- `--watch-rpc`, RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables - `--arb-address`, Address of the deployed arb contract, Will override the 'ARB_ADDRESS' in env variables - `--bot-min-balance` The minimum gas token balance the bot wallet must have. Will override the 'BOT_MIN_BALANCE' in env variables - `-s` or `--subgraph`, Subgraph URL(s) to read orders details from, can be used in combination with --orders, Will override the 'SUBGRAPH' in env variables @@ -186,9 +185,6 @@ RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.co # Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. WRITE_RPC="" -# RPC URLs to watch for new orders, should support required RPC methods -WATCH_RPC="" - # arb contract address ARB_ADDRESS="0x123..." diff --git a/example.env b/example.env index 2c366156..a024cfad 100644 --- a/example.env +++ b/example.env @@ -14,9 +14,6 @@ RPC_URL="https://polygon-mainnet.g.alchemy.com/v2/{API_KEY}, https://rpc.ankr.co # Option to explicitly use these rpc for write transactions, such as flashbots or mev protect rpc to protect against mev attacks. WRITE_RPC="" -# RPC URLs to watch for new orders, should support required RPC methods -WATCH_RPC="" - # arb contract address ARB_ADDRESS="0x123..." diff --git a/src/cli.ts b/src/cli.ts index aecb3fc6..57aec464 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -72,9 +72,6 @@ const ENV_OPTIONS = { writeRpc: process?.env?.WRITE_RPC ? Array.from(process?.env?.WRITE_RPC.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, - watchRpc: process?.env?.WATCH_RPC - ? Array.from(process?.env?.WATCH_RPC.matchAll(/[^,\s]+/g)).map((v) => v[0]) - : undefined, subgraph: process?.env?.SUBGRAPH ? Array.from(process?.env?.SUBGRAPH.matchAll(/[^,\s]+/g)).map((v) => v[0]) : undefined, @@ -134,10 +131,6 @@ const getOptions = async (argv: any, version?: string) => { "--write-rpc ", "Option to explicitly use for write transactions, such as flashbots or mev protect rpc to protect against mev attacks, Will override the 'WRITE_RPC' in env variables", ) - .option( - "--watch-rpc ", - "RPC URLs to watch for new orders, should support required RPC methods, Will override the 'WATCH_RPC' in env variables", - ) .option( "--timeout ", "Optional seconds to wait for the transaction to mine before disregarding it, Will override the 'TIMEOUT' in env variables", @@ -203,7 +196,6 @@ const getOptions = async (argv: any, version?: string) => { cmdOptions.mnemonic = cmdOptions.mnemonic || getEnv(ENV_OPTIONS.mnemonic); cmdOptions.rpc = cmdOptions.rpc || getEnv(ENV_OPTIONS.rpc); cmdOptions.writeRpc = cmdOptions.writeRpc || getEnv(ENV_OPTIONS.writeRpc); - cmdOptions.watchRpc = cmdOptions.watchRpc || getEnv(ENV_OPTIONS.watchRpc); cmdOptions.arbAddress = cmdOptions.arbAddress || getEnv(ENV_OPTIONS.arbAddress); cmdOptions.genericArbAddress = cmdOptions.genericArbAddress || getEnv(ENV_OPTIONS.genericArbAddress); @@ -362,9 +354,6 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? if (!/^(0x)?[a-fA-F0-9]{64}$/.test(options.key)) throw "invalid wallet private key"; } if (!options.rpc) throw "undefined RPC URL"; - if (!options.watchRpc || !Array.isArray(options.watchRpc) || !options.watchRpc.length) { - throw "undefined watch RPC URL"; - } if (options.writeRpc) { if ( !Array.isArray(options.writeRpc) || @@ -433,7 +422,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? config, orderbooksOwnersProfileMap: await getOrderbookOwnersProfileMapFromSg( ordersDetails, - config.watchClient, + config.viemClient as any as ViemClient, tokens, (options as CliOptions).ownerProfile, ), diff --git a/src/index.ts b/src/index.ts index 2e55873f..298f223a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -182,15 +182,6 @@ export async function getConfig( undefined, config, ); - const watchClient = await createViemClient( - chainId, - options.watchRpc, - false, - undefined, - options.timeout, - undefined, - config, - ); const dataFetcher = await getDataFetcher( viemClient as any as PublicClient, lps, @@ -211,10 +202,8 @@ export async function getConfig( config.dataFetcher = dataFetcher; config.watchedTokens = options.tokens ?? []; config.selfFundOrders = options.selfFundOrders; - config.watchClient = watchClient; config.publicRpc = options.publicRpc; config.walletKey = walletKey; - config.watchRpc = options.watchRpc; config.route = route; config.rpcRecords = rpcRecords; diff --git a/src/types.ts b/src/types.ts index eda595b3..16eee701 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,7 +23,6 @@ export type CliOptions = { key?: string; mnemonic?: string; rpc: string[]; - watchRpc: string[]; writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; @@ -150,7 +149,6 @@ export type BotConfig = { key?: string; mnemonic?: string; rpc: string[]; - watchRpc: string[]; writeRpc?: string[]; arbAddress: string; genericArbAddress?: string; @@ -166,7 +164,6 @@ export type BotConfig = { mainAccount: ViemClient; accounts: ViemClient[]; selfFundOrders?: SelfFundOrder[]; - watchClient: ViemClient; publicRpc: boolean; walletKey: string; route?: "multi" | "single"; diff --git a/test/cli.test.js b/test/cli.test.js index 45534a6f..dd21bf27 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -100,23 +100,6 @@ describe("Test cli", async function () { try { await startup(["", "", "--key", `0x${"0".repeat(64)}`, "--rpc", "some-rpc"]); assert.fail("expected to fail, but resolved"); - } catch (error) { - const expected = "undefined watch RPC URL"; - assert.equal(error, expected); - } - - try { - await startup([ - "", - "", - "--key", - `0x${"0".repeat(64)}`, - "--rpc", - "some-rpc", - "--watch-rpc", - "some-rpc", - ]); - assert.fail("expected to fail, but resolved"); } catch (error) { const expected = "undefined arb contract address"; assert.equal(error, expected); @@ -130,8 +113,6 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", - "--watch-rpc", - "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -153,8 +134,6 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", - "--watch-rpc", - "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -177,8 +156,6 @@ describe("Test cli", async function () { `0x${"0".repeat(64)}`, "--rpc", "some-rpc", - "--watch-rpc", - "some-rpc", "--arb-address", `0x${"0".repeat(64)}`, "--orderbook-address", @@ -200,8 +177,6 @@ describe("Test cli", async function () { "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", "--rpc", "https://rpc.ankr.com/polygon", - "--watch-rpc", - "some-rpc", "--arb-address", `0x${"1".repeat(40)}`, "--orderbook-address", diff --git a/test/orders.test.js b/test/orders.test.js index c9cfed3c..26d6375b 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -1,6 +1,5 @@ const { assert } = require("chai"); const { OrderV3 } = require("../src/abis"); -const { toOrder } = require("../src/watcher"); const mockServer = require("mockttp").getLocal(); const { ethers, viem, network } = require("hardhat"); const ERC20Artifact = require("./abis/ERC20Upgradeable.json"); @@ -11,6 +10,7 @@ const { utils: { hexlify, randomBytes, keccak256 }, } = require("ethers"); const { + toOrder, getOrderPairs, prepareOrdersForRound, getOrderbookOwnersProfileMapFromSg, From 3c2fd82d38e3a70bb24188b180408eac8c5d9f5d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 11 Nov 2024 03:04:35 +0000 Subject: [PATCH 24/32] fix query --- src/index.ts | 4 ++-- src/query.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/index.ts b/src/index.ts index 298f223a..e542ccee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ import { processOrders } from "./processOrders"; import { Context, Span } from "@opentelemetry/api"; import { checkSgStatus, handleSgResults } from "./sg"; import { Tracer } from "@opentelemetry/sdk-trace-base"; -import { getQuery, SgOrder, statusCheckQuery } from "./query"; +import { querySgOrders, SgOrder, statusCheckQuery } from "./query"; import { BotConfig, BundledOrders, CliOptions, RoundReport, SgFilter, RpcRecord } from "./types"; import { getChainConfig, @@ -62,7 +62,7 @@ export async function getOrderDetails( availableSgs.forEach((v) => { if (v && typeof v === "string") promises.push( - getQuery( + querySgOrders( v, sgFilters?.orderHash, sgFilters?.orderOwner, diff --git a/src/query.ts b/src/query.ts index 5973de7e..52450c98 100644 --- a/src/query.ts +++ b/src/query.ts @@ -94,7 +94,7 @@ export function getQueryPaginated( * @param orderbook - orderbook filter * @param timeout - timeout */ -export async function getQuery( +export async function querySgOrders( subgraph: string, orderHash?: string, owner?: string, @@ -142,7 +142,7 @@ export const statusCheckQuery = `{ }`; export const getRemoveOrdersQuery = (start: number, end: number) => { - return `removeOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { + return `{removeOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { order { id owner @@ -175,11 +175,11 @@ export const getRemoveOrdersQuery = (start: number, end: number) => { transaction { timestamp } -}`; +}}`; }; export const getAddOrdersQuery = (start: number, end: number) => { - return `addOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_gt: "${end.toString()}" } }) { + return `{addOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { order { id owner @@ -212,7 +212,7 @@ export const getAddOrdersQuery = (start: number, end: number) => { transaction { timestamp } -}`; +}}`; }; /** From c7466381fb06b9693af272b2db7c7f0ebcaca318 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Tue, 12 Nov 2024 00:10:31 +0000 Subject: [PATCH 25/32] paginated sg orders watch --- DiagOrder.md | 10 ++-- diag/DiagOrder.sol | 7 ++- src/cli.ts | 9 ++-- src/query.ts | 115 +++++++++++++++++++++++++++------------------ 4 files changed, 83 insertions(+), 58 deletions(-) diff --git a/DiagOrder.md b/DiagOrder.md index 6a597d3e..57c4ecd3 100644 --- a/DiagOrder.md +++ b/DiagOrder.md @@ -1,16 +1,14 @@ - you need foundry installed on your machine for this process, as well as having `forge-std` lib for contract dependencies, or if you have nix package manager installed you can enter the default shell where all the deps are available. - go to the `./diag/DiagOrder.sol` file and modify the data with the ones you want ot diag: +- add the rpc url of the evm network +- add the block number at which the debugging should take place. +- replace the `from` address with the transaction sender address, ie msg.sender. - replace the `to` address with the arb contract address on the desired network. - replace the `data` with the calldata without leading 0x taken from otel (hyperdx). - save the file and now you can run the following command to get the traces: ```bash -forge script diag/DiagOrder.sol:DiagOrder -vvvvv --fork-url --fork-block-number --sender +forge script diag/DiagOrder.sol:DiagOrder -vvvvv ``` -- replace the `path` with the location of the saved file in previous step, -- replace the `` with the network rpc url, -- replace the `` with the block number taken from otel spans, -- replace the `
` with the bot's sender address, can be taken from otel spans -- click enter and wait to get the traces after the traces are printed, the desired data can be extraced. here are some examples to explain the steps required to get the desired data: diff --git a/diag/DiagOrder.sol b/diag/DiagOrder.sol index b7c84f36..6a8c5276 100644 --- a/diag/DiagOrder.sol +++ b/diag/DiagOrder.sol @@ -5,10 +5,13 @@ import {Script} from "../lib/forge-std/src/Script.sol"; contract DiagOrder is Script { function run() external { + vm.createSelectFork(""); // rpc url + vm.rollFork(); // block number address to = ; // put arb contract address - address from = ; + address from = ; // sender address + bytes memory data = hex""; // put calldata here without 0x + vm.startPrank(from); - bytes memory data = hex"calldata"; // put calldata here without 0x (bool success, bytes memory result) = to.call(data); (success, result); } diff --git a/src/cli.ts b/src/cli.ts index 57aec464..a3e73acf 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,12 +3,12 @@ import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; -import { getAddOrders, SgOrder } from "./query"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; +import { getAddOrders, getRemoveOrders, SgOrder } from "./query"; import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; @@ -699,8 +699,9 @@ export const main = async (argv: any, version?: string) => { } try { - // check for new orders changes const now = Math.floor(Date.now() / 1000); + + // handle added orders const addOrdersResult = await Promise.allSettled( lastReadOrdersTimestampMap.map((v) => getAddOrders(v.sg, v.lastReadTimestampAdd, now, options.timeout, roundSpan), @@ -720,9 +721,11 @@ export const main = async (argv: any, version?: string) => { ); } } + + // handle removed orders const rmOrdersResult = await Promise.allSettled( lastReadOrdersTimestampMap.map((v) => - getAddOrders( + getRemoveOrders( v.sg, v.lastReadTimestampRemove, now, diff --git a/src/query.ts b/src/query.ts index 52450c98..eb9cf835 100644 --- a/src/query.ts +++ b/src/query.ts @@ -141,8 +141,8 @@ export const statusCheckQuery = `{ } }`; -export const getRemoveOrdersQuery = (start: number, end: number) => { - return `{removeOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { +export const getRemoveOrdersQuery = (start: number, end: number, skip: number) => { + return `{removeOrders(first: 100, skip: ${skip}, where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { order { id owner @@ -178,8 +178,8 @@ export const getRemoveOrdersQuery = (start: number, end: number) => { }}`; }; -export const getAddOrdersQuery = (start: number, end: number) => { - return `{addOrders(where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { +export const getAddOrdersQuery = (start: number, end: number, skip: number) => { + return `{addOrders(first: 100, skip: ${skip}, where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { order { id owner @@ -229,31 +229,42 @@ export async function getRemoveOrders( timeout?: number, span?: Span, ) { + let skip = 0; + const allResults: any[] = []; const removeOrders: NewSgOrder[] = []; - try { - const res = await axios.post( - subgraph, - { query: getRemoveOrdersQuery(startTimestamp, endTimestamp) }, - { headers: { "Content-Type": "application/json" }, timeout }, - ); - if (typeof res?.data?.data?.removeOrders !== "undefined") { - res.data.data.removeOrders.forEach((v: any) => { - if (typeof v?.order?.active === "boolean" && !v.order.active) { - if (!removeOrders.find((e) => e.order.id === v.order.id)) { - removeOrders.push({ - order: v.order as SgOrder, - timestamp: Number(v.transaction.timestamp), - }); - } + for (;;) { + try { + const res = await axios.post( + subgraph, + { query: getRemoveOrdersQuery(startTimestamp, endTimestamp, skip) }, + { headers: { "Content-Type": "application/json" }, timeout }, + ); + if (typeof res?.data?.data?.removeOrders !== "undefined") { + const orders = res.data.data.removeOrders; + allResults.push(...orders); + if (orders.length < 100) { + break; + } else { + skip += 100; } - }); - } else { - span?.addEvent(`Failed to get new removed orders ${subgraph}: invalid response`); - throw "invalid response"; + } else { + break; + } + } catch (error) { + span?.addEvent(errorSnapshot(`Failed to get new removed orders ${subgraph}`, error)); + throw error; } - } catch (error) { - span?.addEvent(errorSnapshot(`Failed to get new removed orders ${subgraph}`, error)); } + allResults.forEach((v) => { + if (typeof v?.order?.active === "boolean" && !v.order.active) { + if (!removeOrders.find((e) => e.order.id === v.order.id)) { + removeOrders.push({ + order: v.order as SgOrder, + timestamp: Number(v.transaction.timestamp), + }); + } + } + }); removeOrders.sort((a, b) => b.timestamp - a.timestamp); return removeOrders; @@ -273,32 +284,42 @@ export async function getAddOrders( timeout?: number, span?: Span, ) { + let skip = 0; + const allResults: any[] = []; const addOrders: NewSgOrder[] = []; - try { - const res = await axios.post( - subgraph, - { query: getAddOrdersQuery(startTimestamp, endTimestamp) }, - { headers: { "Content-Type": "application/json" }, timeout }, - ); - if (typeof res?.data?.data?.addOrders !== "undefined") { - res.data.data.addOrders.forEach((v: any) => { - if (typeof v?.order?.active === "boolean" && v.order.active) { - if (!addOrders.find((e) => e.order.id === v.order.id)) { - addOrders.push({ - order: v.order as SgOrder, - timestamp: Number(v.transaction.timestamp), - }); - } + for (;;) { + try { + const res = await axios.post( + subgraph, + { query: getAddOrdersQuery(startTimestamp, endTimestamp, skip) }, + { headers: { "Content-Type": "application/json" }, timeout }, + ); + if (typeof res?.data?.data?.addOrders !== "undefined") { + const orders = res.data.data.addOrders; + allResults.push(...orders); + if (orders.length < 100) { + break; + } else { + skip += 100; } - }); - } else { - span?.addEvent(`Failed to get new orders ${subgraph}: invalid response`); - throw "invalid response"; + } else { + break; + } + } catch (error) { + span?.addEvent(errorSnapshot(`Failed to get new added orders ${subgraph}`, error)); + throw error; } - } catch (error) { - span?.addEvent(errorSnapshot(`Failed to get new orders from subgraph ${subgraph}`, error)); - throw error; } + allResults.forEach((v) => { + if (typeof v?.order?.active === "boolean" && v.order.active) { + if (!addOrders.find((e) => e.order.id === v.order.id)) { + addOrders.push({ + order: v.order as SgOrder, + timestamp: Number(v.transaction.timestamp), + }); + } + } + }); addOrders.sort((a, b) => b.timestamp - a.timestamp); return addOrders; From 2f9042ae617042cf11768b5dc734db88e1e3ce40 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Wed, 13 Nov 2024 03:08:05 +0000 Subject: [PATCH 26/32] fix --- src/processOrders.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/processOrders.ts b/src/processOrders.ts index d3b107d6..c6efb7a5 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -613,8 +613,8 @@ export async function processPair(args: { // submit the tx let txhash, txUrl; try { - rawtx.nonce = await getNonce(flashbotSigner !== undefined ? flashbotSigner : signer); - if (flashbotSigner !== undefined) { + rawtx.nonce = await getNonce(writeSigner !== undefined ? writeSigner : signer); + if (writeSigner !== undefined) { rawtx.gas = undefined; } txhash = From bde2f606ba582ec1a371da410b0080fc5a2029df Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Sat, 23 Nov 2024 02:34:50 +0000 Subject: [PATCH 27/32] update --- src/cli.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index f31d2c2b..61d6e5f0 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -307,7 +307,7 @@ export const arbRound = async ( let didClear = false; const { reports = [], avgGasCost = undefined } = await clear( config, - ordersDetails, + bundledOrders, tracer, ctx, ); @@ -325,8 +325,7 @@ export const arbRound = async ( } if ( reports.some( - (v) => - v.status === ProcessPairReportStatus.FoundOpportunity && !v.reason, + (v) => v.status === ProcessPairReportStatus.FoundOpportunity && !v.reason, ) ) { didClear = true; From e8691603b3d635b780c3a45eef0376d3da54cf7d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 25 Nov 2024 02:26:02 +0000 Subject: [PATCH 28/32] fix watch orders bug --- src/cli.ts | 59 +++++------ src/query.ts | 274 +++++++++++++++++++++++++-------------------------- 2 files changed, 157 insertions(+), 176 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 61d6e5f0..c5a046d9 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,12 +3,12 @@ import { Command } from "commander"; import { getMetaInfo } from "./config"; import { BigNumber, ethers } from "ethers"; import { Context } from "@opentelemetry/api"; +import { getOrderChanges, SgOrder } from "./query"; import { Resource } from "@opentelemetry/resources"; import { getOrderDetails, clear, getConfig } from "."; import { ErrorSeverity, errorSnapshot } from "./error"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { ProcessPairReportStatus } from "./processOrders"; -import { getAddOrders, getRemoveOrders, SgOrder } from "./query"; import { sleep, getOrdersTokens, isBigNumberish } from "./utils"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { BotConfig, BundledOrders, CliOptions, ViemClient } from "./types"; @@ -571,10 +571,9 @@ export const main = async (argv: any, version?: string) => { } }); - const lastReadOrdersTimestampMap = options.subgraph.map((v) => ({ + const lastReadOrdersMap = options.subgraph.map((v) => ({ sg: v, - lastReadTimestampAdd: lastReadOrdersTimestamp, - lastReadTimestampRemove: lastReadOrdersTimestamp, + skip: 0, })); const day = 24 * 60 * 60 * 1000; let lastGasReset = Date.now() + day; @@ -795,48 +794,40 @@ export const main = async (argv: any, version?: string) => { } try { - const now = Math.floor(Date.now() / 1000); - - // handle added orders - const addOrdersResult = await Promise.allSettled( - lastReadOrdersTimestampMap.map((v) => - getAddOrders(v.sg, v.lastReadTimestampAdd, now, options.timeout, roundSpan), + // handle order changes (add/remove) + roundSpan.setAttribute( + "watch-new-orders", + JSON.stringify({ + hasRead: lastReadOrdersMap, + startTime: lastReadOrdersTimestamp, + }), + ); + const results = await Promise.allSettled( + lastReadOrdersMap.map((v) => + getOrderChanges( + v.sg, + lastReadOrdersTimestamp, + v.skip, + options.timeout, + roundSpan, + ), ), ); - for (let i = 0; i < addOrdersResult.length; i++) { - const res = addOrdersResult[i]; + for (let i = 0; i < results.length; i++) { + const res = results[i]; if (res.status === "fulfilled") { - lastReadOrdersTimestampMap[i].lastReadTimestampAdd = now; + lastReadOrdersMap[i].skip += res.value.count; await handleAddOrderbookOwnersProfileMap( orderbooksOwnersProfileMap, - res.value.map((v) => v.order), + res.value.addOrders.map((v) => v.order), config.viemClient as any as ViemClient, tokens, options.ownerProfile, roundSpan, ); - } - } - - // handle removed orders - const rmOrdersResult = await Promise.allSettled( - lastReadOrdersTimestampMap.map((v) => - getRemoveOrders( - v.sg, - v.lastReadTimestampRemove, - now, - options.timeout, - roundSpan, - ), - ), - ); - for (let i = 0; i < rmOrdersResult.length; i++) { - const res = rmOrdersResult[i]; - if (res.status === "fulfilled") { - lastReadOrdersTimestampMap[i].lastReadTimestampRemove = now; await handleRemoveOrderbookOwnersProfileMap( orderbooksOwnersProfileMap, - res.value.map((v) => v.order), + res.value.removeOrders.map((v) => v.order), roundSpan, ); } diff --git a/src/query.ts b/src/query.ts index eb9cf835..048a0484 100644 --- a/src/query.ts +++ b/src/query.ts @@ -37,6 +37,22 @@ export type NewSgOrder = { timestamp: number; }; +export type SgTx = { + events: SgEvent[]; + timestamp: string; +}; + +export type SgEvent = SgAddRemoveEvent | SgOtherEvent; + +export type SgAddRemoveEvent = { + __typename: "AddOrder" | "RemoveOrder"; + order: SgOrder; +}; + +export type SgOtherEvent = { + __typename: "Withdrawal" | "Deposit"; +}; + /** * Method to get the subgraph query body with optional filters * @param orderHash - The order hash to apply as filter @@ -141,186 +157,160 @@ export const statusCheckQuery = `{ } }`; -export const getRemoveOrdersQuery = (start: number, end: number, skip: number) => { - return `{removeOrders(first: 100, skip: ${skip}, where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { - order { - id - owner - orderHash - orderBytes - active - nonce - orderbook { - id - } - inputs { - balance - vaultId - token { - address - decimals - symbol +/** + * Get query for transactions + */ +export const getTxsQuery = (start: number, skip: number) => { + return `{transactions( + orderBy: timestamp + orderDirection: desc + first: 100 + skip: ${skip} + where: {timestamp_gt: "${start.toString()}"} + ) { + events { + __typename + ... on AddOrder { + transaction { + timestamp } - } - outputs { - balance - vaultId - token { - address - decimals - symbol + order { + id + owner + orderHash + orderBytes + active + nonce + orderbook { + id + } + inputs { + balance + vaultId + token { + address + decimals + symbol + } + } + outputs { + balance + vaultId + token { + address + decimals + symbol + } + } } } - } - transaction { - timestamp - } -}}`; -}; - -export const getAddOrdersQuery = (start: number, end: number, skip: number) => { - return `{addOrders(first: 100, skip: ${skip}, where: { transaction_: { timestamp_gt: "${start.toString()}", timestamp_lte: "${end.toString()}" } }) { - order { - id - owner - orderHash - orderBytes - active - nonce - orderbook { - id - } - inputs { - balance - vaultId - token { - address - decimals - symbol + ... on RemoveOrder { + transaction { + timestamp } - } - outputs { - balance - vaultId - token { - address - decimals - symbol + order { + id + owner + orderHash + orderBytes + active + nonce + orderbook { + id + } + inputs { + balance + vaultId + token { + address + decimals + symbol + } + } + outputs { + balance + vaultId + token { + address + decimals + symbol + } + } } } } - transaction { - timestamp - } + timestamp }}`; }; /** - * Fecthes the remove orders from the given subgraph in the given timeframe + * Fecthes the order changes after the given time and skipping the first skip txs * @param subgraph - The subgraph url * @param startTimestamp - start timestamp range - * @param endTimestamp - end timestamp range + * @param skip - skip count * @param timeout - promise timeout */ -export async function getRemoveOrders( +export async function getOrderChanges( subgraph: string, startTimestamp: number, - endTimestamp: number, + skip: number, timeout?: number, span?: Span, ) { - let skip = 0; - const allResults: any[] = []; + let skip_ = skip; + let count = 0; + const allResults: SgTx[] = []; + const addOrders: NewSgOrder[] = []; const removeOrders: NewSgOrder[] = []; for (;;) { try { const res = await axios.post( subgraph, - { query: getRemoveOrdersQuery(startTimestamp, endTimestamp, skip) }, + { query: getTxsQuery(startTimestamp, skip_) }, { headers: { "Content-Type": "application/json" }, timeout }, ); - if (typeof res?.data?.data?.removeOrders !== "undefined") { - const orders = res.data.data.removeOrders; - allResults.push(...orders); - if (orders.length < 100) { + if (typeof res?.data?.data?.transactions !== "undefined") { + const txs = res.data.data.transactions; + count += txs.length; + allResults.push(...txs); + if (txs.length < 100) { break; } else { - skip += 100; + skip_ += 100; } } else { break; } } catch (error) { - span?.addEvent(errorSnapshot(`Failed to get new removed orders ${subgraph}`, error)); + span?.addEvent(errorSnapshot(`Failed to get order changes ${subgraph}`, error)); throw error; } } - allResults.forEach((v) => { - if (typeof v?.order?.active === "boolean" && !v.order.active) { - if (!removeOrders.find((e) => e.order.id === v.order.id)) { - removeOrders.push({ - order: v.order as SgOrder, - timestamp: Number(v.transaction.timestamp), - }); - } - } - }); - - removeOrders.sort((a, b) => b.timestamp - a.timestamp); - return removeOrders; -} - -/** - * Fecthes the add orders from the given subgraph in the given timeframe - * @param subgraph - The subgraph url - * @param startTimestamp - start timestamp range - * @param endTimestamp - end timestamp range - * @param timeout - promise timeout - */ -export async function getAddOrders( - subgraph: string, - startTimestamp: number, - endTimestamp: number, - timeout?: number, - span?: Span, -) { - let skip = 0; - const allResults: any[] = []; - const addOrders: NewSgOrder[] = []; - for (;;) { - try { - const res = await axios.post( - subgraph, - { query: getAddOrdersQuery(startTimestamp, endTimestamp, skip) }, - { headers: { "Content-Type": "application/json" }, timeout }, - ); - if (typeof res?.data?.data?.addOrders !== "undefined") { - const orders = res.data.data.addOrders; - allResults.push(...orders); - if (orders.length < 100) { - break; - } else { - skip += 100; + allResults.forEach((tx) => { + if (tx?.events?.length) { + tx.events.forEach((event) => { + if (event.__typename === "AddOrder") { + if (typeof event?.order?.active === "boolean" && event.order.active) { + if (!addOrders.find((e) => e.order.id === event.order.id)) { + addOrders.unshift({ + order: event.order as SgOrder, + timestamp: Number(tx.timestamp), + }); + } + } } - } else { - break; - } - } catch (error) { - span?.addEvent(errorSnapshot(`Failed to get new added orders ${subgraph}`, error)); - throw error; - } - } - allResults.forEach((v) => { - if (typeof v?.order?.active === "boolean" && v.order.active) { - if (!addOrders.find((e) => e.order.id === v.order.id)) { - addOrders.push({ - order: v.order as SgOrder, - timestamp: Number(v.transaction.timestamp), - }); - } + if (event.__typename === "RemoveOrder") { + if (typeof event?.order?.active === "boolean" && !event.order.active) { + if (!removeOrders.find((e) => e.order.id === event.order.id)) { + removeOrders.unshift({ + order: event.order as SgOrder, + timestamp: Number(tx.timestamp), + }); + } + } + } + }); } }); - - addOrders.sort((a, b) => b.timestamp - a.timestamp); - return addOrders; + return { addOrders, removeOrders, count }; } From 6b45f1300926095d0ffb0edc444916b19ff9bd49 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 25 Nov 2024 03:33:54 +0000 Subject: [PATCH 29/32] diag --- src/cli.ts | 36 +++++++++++++++++++++++------------- src/order.ts | 2 ++ src/query.ts | 8 ++++++++ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index c5a046d9..92dc2a07 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -817,19 +817,29 @@ export const main = async (argv: any, version?: string) => { const res = results[i]; if (res.status === "fulfilled") { lastReadOrdersMap[i].skip += res.value.count; - await handleAddOrderbookOwnersProfileMap( - orderbooksOwnersProfileMap, - res.value.addOrders.map((v) => v.order), - config.viemClient as any as ViemClient, - tokens, - options.ownerProfile, - roundSpan, - ); - await handleRemoveOrderbookOwnersProfileMap( - orderbooksOwnersProfileMap, - res.value.removeOrders.map((v) => v.order), - roundSpan, - ); + try { + await handleAddOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap, + res.value.addOrders.map((v) => v.order), + config.viemClient as any as ViemClient, + tokens, + options.ownerProfile, + roundSpan, + ); + } catch { + // eslint-disable-next-line no-console + console.log("bad1"); + } + try { + await handleRemoveOrderbookOwnersProfileMap( + orderbooksOwnersProfileMap, + res.value.removeOrders.map((v) => v.order), + roundSpan, + ); + } catch { + // eslint-disable-next-line no-console + console.log("bad2"); + } } } } catch { diff --git a/src/order.ts b/src/order.ts index bf9af1c0..ba4b8b1c 100644 --- a/src/order.ts +++ b/src/order.ts @@ -211,6 +211,8 @@ export async function handleRemoveOrderbookOwnersProfileMap( ordersDetails: SgOrder[], span?: Span, ) { + // eslint-disable-next-line no-console + console.log("bruh"); const changes: Record = {}; for (let i = 0; i < ordersDetails.length; i++) { const orderDetails = ordersDetails[i]; diff --git a/src/query.ts b/src/query.ts index 048a0484..ea018c45 100644 --- a/src/query.ts +++ b/src/query.ts @@ -300,8 +300,14 @@ export async function getOrderChanges( } } if (event.__typename === "RemoveOrder") { + // eslint-disable-next-line no-console + console.log("abcd"); if (typeof event?.order?.active === "boolean" && !event.order.active) { + // eslint-disable-next-line no-console + console.log("abcd1"); if (!removeOrders.find((e) => e.order.id === event.order.id)) { + // eslint-disable-next-line no-console + console.log("abcd2"); removeOrders.unshift({ order: event.order as SgOrder, timestamp: Number(tx.timestamp), @@ -312,5 +318,7 @@ export async function getOrderChanges( }); } }); + // eslint-disable-next-line no-console + console.log(removeOrders); return { addOrders, removeOrders, count }; } From 10dd57852ff7f6840642bd1176f567a8328ef95d Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 25 Nov 2024 03:58:32 +0000 Subject: [PATCH 30/32] Update order.ts --- src/order.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/order.ts b/src/order.ts index ba4b8b1c..628382db 100644 --- a/src/order.ts +++ b/src/order.ts @@ -231,14 +231,20 @@ export async function handleRemoveOrderbookOwnersProfileMap( } const orderbookOwnerProfileItem = orderbooksOwnersProfileMap.get(orderbook); if (orderbookOwnerProfileItem) { + // eslint-disable-next-line no-console + console.log("aa1"); const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); if (ownerProfile) { - ownerProfile.orders.delete(orderDetails.orderHash.toLowerCase()); + const x = ownerProfile.orders.delete(orderDetails.orderHash.toLowerCase()); + // eslint-disable-next-line no-console + console.log("aa2", x); } } } if (span) { for (const orderbook in changes) { + // eslint-disable-next-line no-console + console.log("aa3"); span.setAttribute(`orderbooksChanges.${orderbook}.removedOrders`, changes[orderbook]); } } From 6cf7bacc06e933827ef9009bcadd73a8e1145f08 Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 25 Nov 2024 04:26:49 +0000 Subject: [PATCH 31/32] fix watcher --- src/cli.ts | 6 ++---- src/order.ts | 10 +--------- src/query.ts | 16 ++++------------ 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 92dc2a07..73a6fddc 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -827,8 +827,7 @@ export const main = async (argv: any, version?: string) => { roundSpan, ); } catch { - // eslint-disable-next-line no-console - console.log("bad1"); + /**/ } try { await handleRemoveOrderbookOwnersProfileMap( @@ -837,8 +836,7 @@ export const main = async (argv: any, version?: string) => { roundSpan, ); } catch { - // eslint-disable-next-line no-console - console.log("bad2"); + /**/ } } } diff --git a/src/order.ts b/src/order.ts index 628382db..bf9af1c0 100644 --- a/src/order.ts +++ b/src/order.ts @@ -211,8 +211,6 @@ export async function handleRemoveOrderbookOwnersProfileMap( ordersDetails: SgOrder[], span?: Span, ) { - // eslint-disable-next-line no-console - console.log("bruh"); const changes: Record = {}; for (let i = 0; i < ordersDetails.length; i++) { const orderDetails = ordersDetails[i]; @@ -231,20 +229,14 @@ export async function handleRemoveOrderbookOwnersProfileMap( } const orderbookOwnerProfileItem = orderbooksOwnersProfileMap.get(orderbook); if (orderbookOwnerProfileItem) { - // eslint-disable-next-line no-console - console.log("aa1"); const ownerProfile = orderbookOwnerProfileItem.get(orderStruct.owner.toLowerCase()); if (ownerProfile) { - const x = ownerProfile.orders.delete(orderDetails.orderHash.toLowerCase()); - // eslint-disable-next-line no-console - console.log("aa2", x); + ownerProfile.orders.delete(orderDetails.orderHash.toLowerCase()); } } } if (span) { for (const orderbook in changes) { - // eslint-disable-next-line no-console - console.log("aa3"); span.setAttribute(`orderbooksChanges.${orderbook}.removedOrders`, changes[orderbook]); } } diff --git a/src/query.ts b/src/query.ts index ea018c45..19f43c32 100644 --- a/src/query.ts +++ b/src/query.ts @@ -163,10 +163,10 @@ export const statusCheckQuery = `{ export const getTxsQuery = (start: number, skip: number) => { return `{transactions( orderBy: timestamp - orderDirection: desc + orderDirection: asc first: 100 skip: ${skip} - where: {timestamp_gt: "${start.toString()}"} + where: { timestamp_gt: "${start}" } ) { events { __typename @@ -292,7 +292,7 @@ export async function getOrderChanges( if (event.__typename === "AddOrder") { if (typeof event?.order?.active === "boolean" && event.order.active) { if (!addOrders.find((e) => e.order.id === event.order.id)) { - addOrders.unshift({ + addOrders.push({ order: event.order as SgOrder, timestamp: Number(tx.timestamp), }); @@ -300,15 +300,9 @@ export async function getOrderChanges( } } if (event.__typename === "RemoveOrder") { - // eslint-disable-next-line no-console - console.log("abcd"); if (typeof event?.order?.active === "boolean" && !event.order.active) { - // eslint-disable-next-line no-console - console.log("abcd1"); if (!removeOrders.find((e) => e.order.id === event.order.id)) { - // eslint-disable-next-line no-console - console.log("abcd2"); - removeOrders.unshift({ + removeOrders.push({ order: event.order as SgOrder, timestamp: Number(tx.timestamp), }); @@ -318,7 +312,5 @@ export async function getOrderChanges( }); } }); - // eslint-disable-next-line no-console - console.log(removeOrders); return { addOrders, removeOrders, count }; } From bee81c00624eccf86154804134c7011ec1f295bb Mon Sep 17 00:00:00 2001 From: rouzwelt Date: Mon, 25 Nov 2024 22:17:47 +0000 Subject: [PATCH 32/32] update --- README.md | 4 +-- example.env | 2 +- src/account.ts | 27 ++++++++++++++----- src/cli.ts | 6 ++--- src/error.ts | 3 +++ src/order.ts | 57 +++++++++++++++++++++------------------ src/processOrders.ts | 24 +++++++---------- test/orders.test.js | 63 +++++++++++++++++++++++++++++++++++++++----- 8 files changed, 126 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index ce610a66..19376695 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Other optional arguments are: - `--owner-profile`, Specifies the owner limit, example: --owner-profile 0x123456=12 . Will override the 'OWNER_PROFILE' in env variables - `--public-rpc`, Allows to use public RPCs as fallbacks, default is false. Will override the 'PUBLIC_RPC' in env variables - `--gas-price-multiplier`, Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7%. Will override the 'GAS_PRICE_MULTIPLIER' in env variables -- `--gas-limit-multiplier`, Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables +- `--gas-limit-multiplier`, Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables - `--tx-gas`, Option to set a static gas limit for all submitting txs. Will override the 'TX_GAS' in env variables - `-V` or `--version`, output the version number - `-h` or `--help`, output usage information @@ -262,7 +262,7 @@ ROUTE="single" # Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7% GAS_PRICE_MULTIPLIER= -# Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5% +# Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8% GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs diff --git a/example.env b/example.env index 61d8b14a..c3b9a355 100644 --- a/example.env +++ b/example.env @@ -88,7 +88,7 @@ ROUTE="single" # Option to multiply the gas price fetched from the rpc as percentage, default is 107, ie +7% GAS_PRICE_MULTIPLIER= -# Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5% +# Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8% GAS_LIMIT_MULTIPLIER= # Option to set a static gas limit for all submitting txs diff --git a/src/account.ts b/src/account.ts index 0185e385..e78797e8 100644 --- a/src/account.ts +++ b/src/account.ts @@ -525,11 +525,11 @@ export async function sweepToMainWallet( balance, ]) as `0x${string}`, }; - txs.push({ tx, bounty, balance: ethers.utils.formatUnits(balance, bounty.decimals) }); const gas = await fromWallet.estimateGas(tx); + txs.push({ tx, bounty, balance: ethers.utils.formatUnits(balance, bounty.decimals) }); cumulativeGasLimit = cumulativeGasLimit.add(gas); } catch { - failedBounties.push(bounty); + addWatchedToken(bounty, failedBounties); } } @@ -612,7 +612,7 @@ export async function sweepToMainWallet( code: SpanStatusCode.ERROR, message: "Failed to sweep back to main wallet: tx reverted", }); - failedBounties.push(txs[i].bounty); + addWatchedToken(txs[i].bounty, failedBounties); } fromWallet.BALANCE = fromWallet.BALANCE.sub(txCost); } catch (error) { @@ -621,14 +621,14 @@ export async function sweepToMainWallet( code: SpanStatusCode.ERROR, message: "Failed to sweep back to main wallet: " + errorSnapshot("", error), }); - failedBounties.push(txs[i].bounty); + addWatchedToken(txs[i].bounty, failedBounties); } span?.end(); } // empty gas if all tokens are swept if (!failedBounties.length) { - const span = tracer?.startSpan("sweep-gas-to-main-wallet", undefined, mainCtx); + const span = tracer?.startSpan("sweep-remaining-gas-to-main-wallet", undefined, mainCtx); span?.setAttribute("details.wallet", fromWallet.account.address); try { const gasLimit = ethers.BigNumber.from( @@ -889,7 +889,22 @@ export async function sweepToEth(config: BotConfig, tracer?: Tracer, ctx?: Conte } export async function setWatchedTokens(account: ViemClient, watchedTokens: TokenDetails[]) { - account.BOUNTY = watchedTokens; + account.BOUNTY = [...watchedTokens]; +} + +export function addWatchedToken( + token: TokenDetails, + watchedTokens: TokenDetails[], + account?: ViemClient, +) { + if (!watchedTokens.find((v) => v.address.toLowerCase() === token.address.toLowerCase())) { + watchedTokens.push(token); + } + if (account) { + if (!account.BOUNTY.find((v) => v.address.toLowerCase() === token.address.toLowerCase())) { + account.BOUNTY.push(token); + } + } } /** diff --git a/src/cli.ts b/src/cli.ts index 73a6fddc..591c82e5 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -194,7 +194,7 @@ const getOptions = async (argv: any, version?: string) => { ) .option( "--gas-limit-multiplier ", - "Option to multiply the gas limit estimation from the rpc as percentage, default is 105, ie +5%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables", + "Option to multiply the gas limit estimation from the rpc as percentage, default is 108, ie +8%. Will override the 'GAS_LIMIT_MULTIPLIER' in env variables", ) .option( "--tx-gas ", @@ -446,7 +446,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? throw "invalid gasLimitMultiplier value, must be an integer greater than zero"; } else throw "invalid gasLimitMultiplier value, must be an integer greater than zero"; } else { - options.gasLimitMultiplier = 105; + options.gasLimitMultiplier = 108; } if (options.txGas) { if (typeof options.txGas === "number") { @@ -478,7 +478,7 @@ export async function startup(argv: any, version?: string, tracer?: Tracer, ctx? } const lastReadOrdersTimestamp = Math.floor(Date.now() / 1000); const tokens = getOrdersTokens(ordersDetails); - options.tokens = [...tokens]; + options.tokens = tokens; // get config const config = await getConfig( diff --git a/src/error.ts b/src/error.ts index 14c734b9..40d02208 100644 --- a/src/error.ts +++ b/src/error.ts @@ -102,9 +102,12 @@ export function errorSnapshot(header: string, err: any): string { export function containsNodeError(err: BaseError): boolean { try { const snapshot = errorSnapshot("", err); + const parsed = parseRevertError(err); return ( // err instanceof TransactionRejectedRpcError || // err instanceof InvalidInputRpcError || + !!parsed.decoded || + !!parsed.raw.data || err instanceof FeeCapTooLowError || err instanceof ExecutionRevertedError || err instanceof InsufficientFundsError || diff --git a/src/order.ts b/src/order.ts index bf9af1c0..d27bf8c0 100644 --- a/src/order.ts +++ b/src/order.ts @@ -2,6 +2,7 @@ import { OrderV3 } from "./abis"; import { SgOrder } from "./query"; import { Span } from "@opentelemetry/api"; import { hexlify } from "ethers/lib/utils"; +import { addWatchedToken } from "./account"; import { getTokenSymbol, shuffleArray } from "./utils"; import { decodeAbiParameters, parseAbiParameters } from "viem"; import { @@ -67,13 +68,14 @@ export async function getOrderPairs( _outputSymbol = symbol; } } else { - if (!tokens.find((v) => v.address.toLowerCase() === _output.token.toLowerCase())) { - tokens.push({ + addWatchedToken( + { address: _output.token.toLowerCase(), symbol: _outputSymbol, decimals: _output.decimals, - }); - } + }, + tokens, + ); } for (let k = 0; k < orderStruct.validInputs.length; k++) { @@ -91,13 +93,14 @@ export async function getOrderPairs( _inputSymbol = symbol; } } else { - if (!tokens.find((v) => v.address.toLowerCase() === _input.token.toLowerCase())) { - tokens.push({ + addWatchedToken( + { address: _input.token.toLowerCase(), symbol: _inputSymbol, decimals: _input.decimals, - }); - } + }, + tokens, + ); } if (_input.token.toLowerCase() !== _output.token.toLowerCase()) @@ -278,29 +281,30 @@ export function prepareOrdersForRound( for (const [, ownerProfile] of ownersProfileMap) { let remainingLimit = ownerProfile.limit; const activeOrdersProfiles = Array.from(ownerProfile.orders).filter((v) => v[1].active); - const remainingOrdersPairs = activeOrdersProfiles.filter( + let remainingOrdersPairs = activeOrdersProfiles.filter( (v) => v[1].takeOrders.length > 0, ); + // reset if all orders are already consumed if (remainingOrdersPairs.length === 0) { - for (const [orderHash, orderProfile] of activeOrdersProfiles) { + for (const [, orderProfile] of activeOrdersProfiles) { orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); - if (remainingLimit > 0) { - const consumingOrderPairs = orderProfile.takeOrders.splice( - 0, - remainingLimit, - ); - remainingLimit -= consumingOrderPairs.length; - orderProfile.consumedTakeOrders.push(...consumingOrderPairs); - gatherPairs( - orderbook, - orderHash, - consumingOrderPairs, - orderbookBundledOrders, - ); - } } - } else { - for (const [orderHash, orderProfile] of remainingOrdersPairs) { + remainingOrdersPairs = activeOrdersProfiles; + } + // consume orders limits + for (const [orderHash, orderProfile] of remainingOrdersPairs) { + if (remainingLimit > 0) { + const consumingOrderPairs = orderProfile.takeOrders.splice(0, remainingLimit); + remainingLimit -= consumingOrderPairs.length; + orderProfile.consumedTakeOrders.push(...consumingOrderPairs); + gatherPairs(orderbook, orderHash, consumingOrderPairs, orderbookBundledOrders); + } + } + // if all orders are consumed and still there is limit remaining, + // reset and start consuming again from top until limit is reached + if (remainingLimit > 0) { + for (const [orderHash, orderProfile] of activeOrdersProfiles) { + orderProfile.takeOrders.push(...orderProfile.consumedTakeOrders.splice(0)); if (remainingLimit > 0) { const consumingOrderPairs = orderProfile.takeOrders.splice( 0, @@ -351,6 +355,7 @@ function gatherPairs( v.sellToken.toLowerCase() === pair.sellToken.toLowerCase(), ); if (bundleOrder) { + // make sure to not duplicate if ( !bundleOrder.takeOrders.find((v) => v.id.toLowerCase() === orderHash.toLowerCase()) ) { diff --git a/src/processOrders.ts b/src/processOrders.ts index d9eac755..33849abc 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -8,7 +8,7 @@ import { privateKeyToAccount } from "viem/accounts"; import { BigNumber, Contract, ethers } from "ethers"; import { Tracer } from "@opentelemetry/sdk-trace-base"; import { Context, SpanStatusCode } from "@opentelemetry/api"; -import { fundOwnedOrders, getNonce, rotateAccounts } from "./account"; +import { addWatchedToken, fundOwnedOrders, getNonce, rotateAccounts } from "./account"; import { containsNodeError, ErrorSeverity, errorSnapshot, handleRevert, isTimeout } from "./error"; import { Report, @@ -762,27 +762,21 @@ export async function processPair(args: { // keep track of gas consumption of the account and bounty token result.gasCost = actualGasCost; - if ( - inputTokenIncome && - inputTokenIncome.gt(0) && - !signer.BOUNTY.find((v) => v.address === orderPairObject.buyToken) - ) { - signer.BOUNTY.push({ + if (inputTokenIncome && inputTokenIncome.gt(0)) { + const tkn = { address: orderPairObject.buyToken.toLowerCase(), decimals: orderPairObject.buyTokenDecimals, symbol: orderPairObject.buyTokenSymbol, - }); + }; + addWatchedToken(tkn, config.watchedTokens ?? [], signer); } - if ( - outputTokenIncome && - outputTokenIncome.gt(0) && - !signer.BOUNTY.find((v) => v.address === orderPairObject.sellToken) - ) { - signer.BOUNTY.push({ + if (outputTokenIncome && outputTokenIncome.gt(0)) { + const tkn = { address: orderPairObject.sellToken.toLowerCase(), decimals: orderPairObject.sellTokenDecimals, symbol: orderPairObject.sellTokenSymbol, - }); + }; + addWatchedToken(tkn, config.watchedTokens ?? [], signer); } return result; } else { diff --git a/test/orders.test.js b/test/orders.test.js index 26d6375b..a2ea81ff 100644 --- a/test/orders.test.js +++ b/test/orders.test.js @@ -753,7 +753,7 @@ describe("Test order details", async function () { [order1, order2, order3, order4, order5, order6, order7, order8], undefined, [], - { [owner1]: 3, [owner2]: 1 }, // set owner1 limit as 3, owner2 to 1 + { [owner1]: 4, [owner2]: 1 }, // set owner1 limit as 4, owner2 to 1 ); // prepare orders for first round @@ -768,8 +768,8 @@ describe("Test order details", async function () { sellTokenSymbol: token2.symbol, sellTokenDecimals: token2.decimals, orderbook, - // first 3 owner1 orders for round1, owner1 limit is 3 - takeOrders: owner1Orders.slice(0, 3).map((v) => ({ + // first 4 owner1 orders for round1, owner1 limit is 4 + takeOrders: owner1Orders.slice(0, 4).map((v) => ({ id: v.id, takeOrder: { order: v.struct, @@ -814,8 +814,11 @@ describe("Test order details", async function () { sellTokenSymbol: token2.symbol, sellTokenDecimals: token2.decimals, orderbook, - // second 3 owner1 orders for round2, owner1 limit is 3 - takeOrders: owner1Orders.slice(3, owner1Orders.length).map((v) => ({ + // first2 and last 2 owner1 orders for round2, owner1 limit is 4 + takeOrders: [ + ...owner1Orders.slice(4, owner1Orders.length), + ...owner1Orders.slice(0, 2), + ].map((v) => ({ id: v.id, takeOrder: { order: v.struct, @@ -861,8 +864,8 @@ describe("Test order details", async function () { sellTokenSymbol: token2.symbol, sellTokenDecimals: token2.decimals, orderbook, - // first 3 owner1 orders again for round3, owner1 limit is 3 - takeOrders: owner1Orders.slice(0, 3).map((v) => ({ + // last 4 owner1 orders again for round3, owner1 limit is 4 + takeOrders: owner1Orders.slice(2).map((v) => ({ id: v.id, takeOrder: { order: v.struct, @@ -894,6 +897,52 @@ describe("Test order details", async function () { ], ]; assert.deepEqual(result3, expected3); + + // prepare orders for 4th round + const result4 = prepareOrdersForRound(allOrders, false); + const expected4 = [ + [ + { + buyToken: token1.address, + buyTokenSymbol: token1.symbol, + buyTokenDecimals: token1.decimals, + sellToken: token2.address, + sellTokenSymbol: token2.symbol, + sellTokenDecimals: token2.decimals, + orderbook, + // back to first 4 owner1 orders for round4, owner1 limit is 4 + takeOrders: owner1Orders.slice(0, 4).map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + { + buyToken: token2.address, + buyTokenSymbol: token2.symbol, + buyTokenDecimals: token2.decimals, + sellToken: token1.address, + sellTokenSymbol: token1.symbol, + sellTokenDecimals: token1.decimals, + orderbook, + // second 1 owner2 orders for round4, owner2 limit is 1 + takeOrders: owner2Orders.slice(1).map((v) => ({ + id: v.id, + takeOrder: { + order: v.struct, + inputIOIndex: 0, + outputIOIndex: 0, + signedContext: [], + }, + })), + }, + ], + ]; + assert.deepEqual(result4, expected4); }); });