diff --git a/src/modes/index.ts b/src/modes/index.ts index 433d8dfd..171e1c5f 100644 --- a/src/modes/index.ts +++ b/src/modes/index.ts @@ -117,24 +117,18 @@ export async function findOpp({ noneNodeError: undefined, }; if ((allResults[0] as any)?.reason?.spanAttributes) { - extendSpanAttributes( - spanAttributes, + spanAttributes["routeProcessor"] = JSON.stringify( (allResults[0] as any).reason.spanAttributes, - "routeProcessor", ); } if ((allResults[1] as any)?.reason?.spanAttributes) { - extendSpanAttributes( - spanAttributes, - (allResults[1] as any).reason.spanAttributes, - "intraOrderbook", + spanAttributes["intraOrderbook"] = JSON.stringify( + (allResults[1] as any).reason.spanAttributes["intraOrderbook"], ); } if ((allResults[2] as any)?.reason?.spanAttributes) { - extendSpanAttributes( - spanAttributes, + spanAttributes["interOrderbook"] = JSON.stringify( (allResults[2] as any).reason.spanAttributes, - "interOrderbook", ); } if ((allResults[0] as any)?.reason?.value?.noneNodeError) { @@ -153,3 +147,22 @@ export async function findOpp({ throw result; } } + +/** + * Records gas estimates for otel span attributes + */ +export function recordGasEstAttrs( + spanAttributes: Record, + estimation: any, + config: BotConfig, + headroom: boolean, +) { + const header = headroom ? "headroom" : "final"; + spanAttributes[`gasEst.${header}.gasLimit`] = estimation.gas.toString(); + spanAttributes[`gasEst.${header}.totalCost`] = estimation.totalGasCost.toString(); + spanAttributes[`gasEst.${header}.gasPrice`] = estimation.gasPrice.toString(); + if (config.isSpecialL2) { + spanAttributes[`gasEst.${header}.l1Cost`] = estimation.l1Cost.toString(); + spanAttributes[`gasEst.${header}.l1GasPrice`] = estimation.l1GasPrice.toString(); + } +} diff --git a/src/modes/interOrderbook.ts b/src/modes/interOrderbook.ts index 879fa1d5..8e7b41d5 100644 --- a/src/modes/interOrderbook.ts +++ b/src/modes/interOrderbook.ts @@ -1,17 +1,12 @@ +import { recordGasEstAttrs } from "."; import { orderbookAbi } from "../abis"; import { estimateGasCost } from "../gas"; import { BigNumber, Contract, ethers } from "ethers"; import { containsNodeError, errorSnapshot } from "../error"; import { getBountyEnsureRainlang, parseRainlang } from "../task"; import { BaseError, ExecutionRevertedError, PublicClient } from "viem"; +import { ONE18, scale18To, estimateProfit, withBigintSerializer } from "../utils"; import { BotConfig, BundledOrders, ViemClient, DryrunResult, SpanAttrs } from "../types"; -import { - ONE18, - scale18To, - estimateProfit, - withBigintSerializer, - extendSpanAttributes, -} from "../utils"; const obInterface = new ethers.utils.Interface(orderbookAbi); @@ -128,21 +123,7 @@ export async function dryrun({ gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); // include dryrun headroom gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.headroom", - ); + recordGasEstAttrs(spanAttributes, estimation, config, true); } catch (e) { const isNodeError = containsNodeError(e as BaseError); const errMsg = errorSnapshot("", e); @@ -212,21 +193,7 @@ export async function dryrun({ gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); // include dryrun final gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.final", - ); + recordGasEstAttrs(spanAttributes, estimation, config, false); task.evaluable.bytecode = await parseRainlang( await getBountyEnsureRainlang( ethers.utils.parseUnits(inputToEthPrice), @@ -423,13 +390,12 @@ export async function findOpp({ // } catch { // /**/ // } + const allOrderbooksAttributes: any = {}; for (let i = 0; i < e.errors.length; i++) { - extendSpanAttributes( - spanAttributes, - e.errors[i].spanAttributes, - "againstOrderbooks." + opposingOrderbookOrders[i].orderbook, - ); + allOrderbooksAttributes[opposingOrderbookOrders[i].orderbook] = + e.errors[i].spanAttributes; } + spanAttributes["againstOrderbooks"] = JSON.stringify(allOrderbooksAttributes); const noneNodeErrors = allNoneNodeErrors.filter((v) => !!v); if (allNoneNodeErrors.length && noneNodeErrors.length / allNoneNodeErrors.length > 0.5) { result.value = { diff --git a/src/modes/intraOrderbook.ts b/src/modes/intraOrderbook.ts index 39420088..3c78dc07 100644 --- a/src/modes/intraOrderbook.ts +++ b/src/modes/intraOrderbook.ts @@ -1,10 +1,11 @@ +import { recordGasEstAttrs } from "."; import { orderbookAbi } from "../abis"; import { estimateGasCost } from "../gas"; import { BigNumber, ethers } from "ethers"; import { containsNodeError, errorSnapshot } from "../error"; import { getWithdrawEnsureRainlang, parseRainlang } from "../task"; +import { estimateProfit, scale18, withBigintSerializer } from "../utils"; import { BaseError, erc20Abi, ExecutionRevertedError, PublicClient } from "viem"; -import { estimateProfit, scale18, withBigintSerializer, extendSpanAttributes } from "../utils"; import { SpanAttrs, BotConfig, @@ -120,21 +121,7 @@ export async function dryrun({ gasLimit = ethers.BigNumber.from(estimation.gas).mul(config.gasLimitMultiplier).div(100); // include dryrun headroom gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.headroom", - ); + recordGasEstAttrs(spanAttributes, estimation, config, true); } catch (e) { // reason, code, method, transaction, error, stack, message const isNodeError = containsNodeError(e as BaseError); @@ -214,21 +201,7 @@ export async function dryrun({ gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); // include dryrun final gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.final", - ); + recordGasEstAttrs(spanAttributes, estimation, config, false); task.evaluable.bytecode = await parseRainlang( await getWithdrawEnsureRainlang( signer.account.address, @@ -358,6 +331,7 @@ export async function findOpp({ ); if (!opposingOrders || !opposingOrders.length) throw undefined; + const allErrorAttributes: string[] = []; const allNoneNodeErrors: (string | undefined)[] = []; const inputBalance = scale18( await viemClient.readContract({ @@ -394,9 +368,10 @@ export async function findOpp({ }); } catch (e: any) { allNoneNodeErrors.push(e?.value?.noneNodeError); - extendSpanAttributes(spanAttributes, e.spanAttributes, "intraOrderbook." + i); + allErrorAttributes.push(JSON.stringify(e.spanAttributes)); } } + spanAttributes["intraOrderbook"] = allErrorAttributes; const noneNodeErrors = allNoneNodeErrors.filter((v) => !!v); if (allNoneNodeErrors.length && noneNodeErrors.length / allNoneNodeErrors.length > 0.5) { result.value = { diff --git a/src/modes/routeProcessor.ts b/src/modes/routeProcessor.ts index 331126e9..f30d7270 100644 --- a/src/modes/routeProcessor.ts +++ b/src/modes/routeProcessor.ts @@ -1,3 +1,4 @@ +import { recordGasEstAttrs } from "."; import { Token } from "sushi/currency"; import { estimateGasCost } from "../gas"; import { ChainId, DataFetcher, Router } from "sushi"; @@ -14,7 +15,6 @@ import { estimateProfit, visualizeRoute, withBigintSerializer, - extendSpanAttributes, } from "../utils"; /** @@ -191,21 +191,7 @@ export async function dryrun({ .div(100); // include dryrun headroom gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.headroom", - ); + recordGasEstAttrs(spanAttributes, estimation, config, true); } catch (e) { // reason, code, method, transaction, error, stack, message const isNodeError = containsNodeError(e as BaseError); @@ -277,21 +263,7 @@ export async function dryrun({ gasCost = gasLimit.mul(gasPrice).add(estimation.l1Cost); // include dryrun final gas estimation in otel logs - extendSpanAttributes( - spanAttributes, - { - gasLimit: estimation.gas.toString(), - totalCost: estimation.totalGasCost.toString(), - gasPrice: estimation.gasPrice.toString(), - ...(config.isSpecialL2 - ? { - l1Cost: estimation.l1Cost.toString(), - l1GasPrice: estimation.l1GasPrice.toString(), - } - : {}), - }, - "gasEst.final", - ); + recordGasEstAttrs(spanAttributes, estimation, config, false); task.evaluable.bytecode = await parseRainlang( await getBountyEnsureRainlang( ethers.utils.parseUnits(ethPrice), @@ -430,7 +402,7 @@ export async function findOpp({ // the fail reason can only be no route in case all hops fail reasons are no route if (e.reason !== RouteProcessorDryrunHaltReason.NoRoute) noRoute = false; allNoneNodeErrors.push(e?.value?.noneNodeError); - extendSpanAttributes(spanAttributes, e.spanAttributes, "full"); + spanAttributes["full"] = JSON.stringify(e.spanAttributes); } if (!hasPriceMatch.value) { const maxTradeSize = findMaxInput({ @@ -463,7 +435,7 @@ export async function findOpp({ // the fail reason can only be no route in case all hops fail reasons are no route if (e.reason !== RouteProcessorDryrunHaltReason.NoRoute) noRoute = false; allNoneNodeErrors.push(e?.value?.noneNodeError); - extendSpanAttributes(spanAttributes, e.spanAttributes, "partial"); + spanAttributes["partial"] = JSON.stringify(e.spanAttributes); } } } diff --git a/src/processOrders.ts b/src/processOrders.ts index e895e6cc..336a394f 100644 --- a/src/processOrders.ts +++ b/src/processOrders.ts @@ -201,6 +201,9 @@ export const processOrders = async ( } for (const { settle, pair, orderPairObject } of results) { + // instantiate a span for this pair + const span = tracer.startSpan(`order_${pair}`, undefined, ctx); + span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); try { // settle the process results // this will return the report of the operation and in case @@ -208,10 +211,6 @@ export const processOrders = async ( // the root cause as well const result = await settle(); - // instantiate a span for this pair - const span = tracer.startSpan(`order_${pair}`, undefined, ctx); - span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); - // keep track of avg gas cost if (result.gasCost) { txGasCosts.push(result.gasCost); @@ -238,12 +237,7 @@ export const processOrders = async ( span.setAttribute("severity", ErrorSeverity.HIGH); span.setStatus({ code: SpanStatusCode.ERROR, message: "unexpected error" }); } - span.end(); } catch (e: any) { - // instantiate a span for this pair - const span = tracer.startSpan(`order_${pair}`, undefined, ctx); - span.setAttribute("details.owner", orderPairObject.takeOrders[0].takeOrder.order.owner); - // set the span attributes with the values gathered at processPair() span.setAttributes(e.spanAttributes); @@ -366,8 +360,8 @@ export const processOrders = async ( span.setAttribute("severity", ErrorSeverity.HIGH); span.setStatus({ code: SpanStatusCode.ERROR, message }); } - span.end(); } + span.end(); } return { reports, @@ -610,9 +604,40 @@ export async function processPair(args: { } } } catch (e: any) { - // record all span attributes + // record all span attributes in their scopes for (const attrKey in e.spanAttributes) { - spanAttributes["details." + attrKey] = e.spanAttributes[attrKey]; + if (attrKey === "routeProcessor") { + const rpAttrs = JSON.parse(e.spanAttributes[attrKey]); + for (const key in rpAttrs) { + const innerAttrs = JSON.parse(rpAttrs[key]); + for (const innerKey in innerAttrs) { + spanAttributes["details.routeProcessor." + key + "." + innerKey] = + innerAttrs[innerKey]; + } + } + } else if (attrKey === "intraOrderbook") { + const intraAttrs = JSON.parse(e.spanAttributes[attrKey]); + for (let i = 0; i < intraAttrs.length; i++) { + const innerAttrs = JSON.parse(intraAttrs[i]); + for (const innerKey in innerAttrs) { + spanAttributes["details.intraOrderbook." + i + "." + innerKey] = + innerAttrs[innerKey]; + } + } + } else if (attrKey === "interOrderbook") { + const interAttrs = JSON.parse( + JSON.parse(e.spanAttributes[attrKey])["againstOrderbooks"], + ); + for (const key in interAttrs) { + for (const innerKey in interAttrs[key]) { + spanAttributes[ + "details.interOrderbook.againstOrderbooks." + key + "." + innerKey + ] = interAttrs[key][innerKey]; + } + } + } else { + spanAttributes["details." + attrKey] = e.spanAttributes[attrKey]; + } } if (e.noneNodeError) { spanAttributes["details.noneNodeError"] = true; diff --git a/test/findOpp.test.js b/test/findOpp.test.js index 1d71152f..57cae604 100644 --- a/test/findOpp.test.js +++ b/test/findOpp.test.js @@ -504,28 +504,31 @@ describe("Test find opp", async function () { oppBlockNumber: undefined, noneNodeError: errorSnapshot("", err), spanAttributes: { - // rp span attrs - "routeProcessor.full.stage": 1, - "routeProcessor.full.rawtx": rawtx, - "routeProcessor.full.isNodeError": false, - "routeProcessor.full.route": expectedRouteVisual, - "routeProcessor.full.blockNumber": oppBlockNumber, - "routeProcessor.full.error": errorSnapshot("", err), - "routeProcessor.full.amountIn": formatUnits(vaultBalance), - "routeProcessor.full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), - "routeProcessor.full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), - - // inter-ob span attrs - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.stage`]: 1, - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.isNodeError`]: false, - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.blockNumber`]: - oppBlockNumber, - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.rawtx`]: - JSON.stringify(rawtx2, withBigintSerializer), - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.maxInput`]: - vaultBalance.toString(), - [`interOrderbook.againstOrderbooks.${opposingOrderbookAddress}.error`]: - errorSnapshot("", err), + routeProcessor: JSON.stringify({ + full: JSON.stringify({ + amountIn: formatUnits(vaultBalance), + amountOut: formatUnits(getAmountOut(vaultBalance), 6), + marketPrice: formatUnits(getCurrentPrice(vaultBalance)), + route: expectedRouteVisual, + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", err), + rawtx: rawtx, + }), + }), + interOrderbook: JSON.stringify({ + againstOrderbooks: JSON.stringify({ + [opposingOrderbookAddress]: { + maxInput: vaultBalance.toString(), + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", err), + rawtx: JSON.stringify(rawtx2, withBigintSerializer), + }, + }), + }), }, }; assert.deepEqual(error, expected); diff --git a/test/mode-interOrderbook.test.js b/test/mode-interOrderbook.test.js index f5b3a198..7991895f 100644 --- a/test/mode-interOrderbook.test.js +++ b/test/mode-interOrderbook.test.js @@ -375,13 +375,16 @@ describe("Test inter-orderbook find opp", async function () { }, reason: undefined, spanAttributes: { - [`againstOrderbooks.${opposingOrderbookAddress}.blockNumber`]: oppBlockNumber, - [`againstOrderbooks.${opposingOrderbookAddress}.stage`]: 1, - [`againstOrderbooks.${opposingOrderbookAddress}.isNodeError`]: false, - [`againstOrderbooks.${opposingOrderbookAddress}.error`]: errorSnapshot("", err), - [`againstOrderbooks.${opposingOrderbookAddress}.rawtx`]: JSON.stringify(rawtx), - [`againstOrderbooks.${opposingOrderbookAddress}.maxInput`]: - vaultBalance.toString(), + againstOrderbooks: JSON.stringify({ + [opposingOrderbookAddress]: { + maxInput: vaultBalance.toString(), + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", err), + rawtx: JSON.stringify(rawtx), + }, + }), }, }; assert.deepEqual(error, expected); diff --git a/test/mode-intraOrderbook.test.js b/test/mode-intraOrderbook.test.js index fd713563..b0d40eee 100644 --- a/test/mode-intraOrderbook.test.js +++ b/test/mode-intraOrderbook.test.js @@ -375,11 +375,15 @@ describe("Test intra-orderbook find opp", async function () { }, reason: undefined, spanAttributes: { - "intraOrderbook.0.blockNumber": oppBlockNumber, - "intraOrderbook.0.stage": 1, - "intraOrderbook.0.isNodeError": false, - "intraOrderbook.0.error": errorSnapshot("", err), - "intraOrderbook.0.rawtx": JSON.stringify(rawtx), + intraOrderbook: [ + JSON.stringify({ + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", err), + rawtx: JSON.stringify(rawtx), + }), + ], }, }; assert.deepEqual(error, expected); diff --git a/test/mode-routeProcessor.test.js b/test/mode-routeProcessor.test.js index 0c983006..7e4f4e75 100644 --- a/test/mode-routeProcessor.test.js +++ b/test/mode-routeProcessor.test.js @@ -532,15 +532,17 @@ describe("Test route processor find opp", async function () { }, reason: RouteProcessorDryrunHaltReason.NoOpportunity, spanAttributes: { - "full.amountIn": formatUnits(vaultBalance), - "full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), - "full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), - "full.route": expectedRouteVisual, - "full.blockNumber": oppBlockNumber, - "full.stage": 1, - "full.isNodeError": false, - "full.error": errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), - "full.rawtx": rawtx, + full: JSON.stringify({ + amountIn: formatUnits(vaultBalance), + amountOut: formatUnits(getAmountOut(vaultBalance), 6), + marketPrice: formatUnits(getCurrentPrice(vaultBalance)), + route: expectedRouteVisual, + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), + rawtx: rawtx, + }), }, }; assert.deepEqual(error, expected); @@ -571,8 +573,10 @@ describe("Test route processor find opp", async function () { value: undefined, reason: RouteProcessorDryrunHaltReason.NoRoute, spanAttributes: { - "full.amountIn": formatUnits(vaultBalance), - "full.route": "no-way", + full: JSON.stringify({ + amountIn: formatUnits(vaultBalance), + route: "no-way", + }), }, }; assert.deepEqual(error, expected); @@ -756,15 +760,17 @@ describe("Test find opp with retries", async function () { }, reason: RouteProcessorDryrunHaltReason.NoOpportunity, spanAttributes: { - "full.amountIn": formatUnits(vaultBalance), - "full.amountOut": formatUnits(getAmountOut(vaultBalance), 6), - "full.marketPrice": formatUnits(getCurrentPrice(vaultBalance)), - "full.route": expectedRouteVisual, - "full.blockNumber": oppBlockNumber, - "full.stage": 1, - "full.isNodeError": false, - "full.error": errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), - "full.rawtx": rawtx, + full: JSON.stringify({ + amountIn: formatUnits(vaultBalance), + amountOut: formatUnits(getAmountOut(vaultBalance), 6), + marketPrice: formatUnits(getCurrentPrice(vaultBalance)), + route: expectedRouteVisual, + blockNumber: oppBlockNumber, + stage: 1, + isNodeError: false, + error: errorSnapshot("", ethers.errors.UNPREDICTABLE_GAS_LIMIT), + rawtx: rawtx, + }), }, }; assert.deepEqual(error, expected);