From 4d9a6e244752ae16a8dd79450edec8af0d8cb4ba Mon Sep 17 00:00:00 2001 From: jsy1218 <91580504+jsy1218@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:21:10 -0400 Subject: [PATCH] feat: add new caching route lambda for async routing invocation --- bin/stacks/routing-lambda-stack.ts | 116 +++++++++++++++++- lib/handlers/injector-sor.ts | 25 +++- .../dynamo-route-caching-provider.ts | 20 +-- 3 files changed, 144 insertions(+), 17 deletions(-) diff --git a/bin/stacks/routing-lambda-stack.ts b/bin/stacks/routing-lambda-stack.ts index d20a3c313..3d1fb4a99 100644 --- a/bin/stacks/routing-lambda-stack.ts +++ b/bin/stacks/routing-lambda-stack.ts @@ -102,6 +102,70 @@ export class RoutingLambdaStack extends cdk.NestedStack { const region = cdk.Stack.of(this).region + const cachingRoutingLambda = new aws_lambda_nodejs.NodejsFunction(this, 'CachingRoutingLambda', { + role: lambdaRole, + runtime: aws_lambda.Runtime.NODEJS_18_X, + entry: path.join(__dirname, '../../lib/handlers/index.ts'), + handler: 'quoteHandler', + // 11/8/23: URA currently calls the Routing API with a timeout of 10 seconds. + // Set this lambda's timeout to be slightly lower to give them time to + // log the response in the event of a failure on our end. + timeout: cdk.Duration.seconds(9), + memorySize: 2560, + deadLetterQueueEnabled: true, + bundling: { + minify: true, + sourceMap: true, + }, + + awsSdkConnectionReuse: true, + + description: 'Caching Routing Lambda', + environment: { + VERSION: '1', + NODE_OPTIONS: '--enable-source-maps', + POOL_CACHE_BUCKET: poolCacheBucket.bucketName, + POOL_CACHE_BUCKET_3: poolCacheBucket3.bucketName, + POOL_CACHE_GZIP_KEY: poolCacheGzipKey, + TOKEN_LIST_CACHE_BUCKET: tokenListCacheBucket.bucketName, + ETH_GAS_STATION_INFO_URL: ethGasStationInfoUrl, + TENDERLY_USER: tenderlyUser, + TENDERLY_PROJECT: tenderlyProject, + TENDERLY_ACCESS_KEY: tenderlyAccessKey, + TENDERLY_NODE_API_KEY: tenderlyNodeApiKey, + // WARNING: Dynamo table name should be the tableinstance.name, e.g. routesDynamoDb.tableName. + // But we tried and had seen lambd version error: + // The following resource(s) failed to create: [RoutingLambda2CurrentVersion49A1BB948389ce4f9c26b15e2ccb07b4c1bab726]. + // 2023-09-01 10:22:43 UTC-0700RoutingLambda2CurrentVersion49A1BB948389ce4f9c26b15e2ccb07b4c1bab726CREATE_FAILED + // A version for this Lambda function exists ( 261 ). Modify the function to create a new version. + // Hence we do not want to modify the table name below. + ROUTES_TABLE_NAME: DynamoDBTableProps.RoutesDbTable.Name, + ROUTES_CACHING_REQUEST_FLAG_TABLE_NAME: DynamoDBTableProps.RoutesDbCachingRequestFlagTable.Name, + CACHED_ROUTES_TABLE_NAME: DynamoDBTableProps.CacheRouteDynamoDbTable.Name, + CACHING_REQUEST_FLAG_TABLE_NAME: DynamoDBTableProps.CachingRequestFlagDynamoDbTable.Name, + CACHED_V3_POOLS_TABLE_NAME: DynamoDBTableProps.V3PoolsDynamoDbTable.Name, + V2_PAIRS_CACHE_TABLE_NAME: DynamoDBTableProps.V2PairsDynamoCache.Name, + RPC_PROVIDER_HEALTH_TABLE_NAME: DynamoDBTableProps.RpcProviderHealthStateDbTable.Name, + + // tokenPropertiesCachingDynamoDb.tableName is the correct format. + // we will start using the correct ones going forward + TOKEN_PROPERTIES_CACHING_TABLE_NAME: tokenPropertiesCachingDynamoDb.tableName, + UNICORN_SECRET: unicornSecret, + GQL_URL: uniGraphQLEndpoint, + GQL_H_ORGN: uniGraphQLHeaderOrigin, + ...jsonRpcProviders, + }, + layers: [ + aws_lambda.LayerVersion.fromLayerVersionArn( + this, + 'InsightsLayer', + `arn:aws:lambda:${region}:580247275435:layer:LambdaInsightsExtension:14` + ), + ], + tracing: aws_lambda.Tracing.ACTIVE, + logRetention: RetentionDays.TWO_WEEKS, + }) + this.routingLambda = new aws_lambda_nodejs.NodejsFunction(this, 'RoutingLambda2', { role: lambdaRole, runtime: aws_lambda.Runtime.NODEJS_18_X, @@ -153,12 +217,13 @@ export class RoutingLambdaStack extends cdk.NestedStack { UNICORN_SECRET: unicornSecret, GQL_URL: uniGraphQLEndpoint, GQL_H_ORGN: uniGraphQLHeaderOrigin, + CACHING_ROUTING_LAMBDA_FUNCTION_NAME: cachingRoutingLambda.functionName, ...jsonRpcProviders, }, layers: [ aws_lambda.LayerVersion.fromLayerVersionArn( this, - 'InsightsLayer', + 'CachingInsightsLayer', `arn:aws:lambda:${region}:580247275435:layer:LambdaInsightsExtension:14` ), ], @@ -166,7 +231,24 @@ export class RoutingLambdaStack extends cdk.NestedStack { logRetention: RetentionDays.TWO_WEEKS, }) - const lambdaAlarmErrorRate = new aws_cloudwatch.Alarm(this, 'RoutingAPI-LambdaErrorRate', { + const cachingLambdaAlarmErrorRate = new aws_cloudwatch.Alarm(this, 'RoutingAPI-LambdaErrorRate', { + metric: new aws_cloudwatch.MathExpression({ + expression: 'errors / invocations', + usingMetrics: { + errors: cachingRoutingLambda.metricErrors({ + period: Duration.minutes(5), + statistic: 'avg', + }), + invocations: cachingRoutingLambda.metricInvocations({ + period: Duration.minutes(5), + statistic: 'avg', + }), + }, + }), + threshold: 0.05, + evaluationPeriods: 3, + }) + const lambdaAlarmErrorRate = new aws_cloudwatch.Alarm(this, 'CachingRoutingAPI-LambdaErrorRate', { metric: new aws_cloudwatch.MathExpression({ expression: 'errors / invocations', usingMetrics: { @@ -184,6 +266,14 @@ export class RoutingLambdaStack extends cdk.NestedStack { evaluationPeriods: 3, }) + const cachingLambdaThrottlesErrorRate = new aws_cloudwatch.Alarm(this, 'CachingRoutingAPI-LambdaThrottles', { + metric: cachingRoutingLambda.metricThrottles({ + period: Duration.minutes(5), + statistic: 'sum', + }), + threshold: 10, + evaluationPeriods: 3, + }) const lambdaThrottlesErrorRate = new aws_cloudwatch.Alarm(this, 'RoutingAPI-LambdaThrottles', { metric: this.routingLambda.metricThrottles({ period: Duration.minutes(5), @@ -196,13 +286,20 @@ export class RoutingLambdaStack extends cdk.NestedStack { if (chatbotSNSArn) { const chatBotTopic = aws_sns.Topic.fromTopicArn(this, 'ChatbotTopic', chatbotSNSArn) + cachingLambdaAlarmErrorRate.addAlarmAction(new aws_cloudwatch_actions.SnsAction(chatBotTopic)) lambdaAlarmErrorRate.addAlarmAction(new aws_cloudwatch_actions.SnsAction(chatBotTopic)) + cachingLambdaThrottlesErrorRate.addAlarmAction(new aws_cloudwatch_actions.SnsAction(chatBotTopic)) lambdaThrottlesErrorRate.addAlarmAction(new aws_cloudwatch_actions.SnsAction(chatBotTopic)) } const enableProvisionedConcurrency = provisionedConcurrency > 0 + const cachingRoutingLambdaAlias = new aws_lambda.Alias(this, 'CachingRoutingLiveAlias', { + aliasName: 'live', + version: cachingRoutingLambda.currentVersion, + provisionedConcurrentExecutions: enableProvisionedConcurrency ? provisionedConcurrency : undefined, + }) this.routingLambdaAlias = new aws_lambda.Alias(this, 'RoutingLiveAlias', { aliasName: 'live', version: this.routingLambda.currentVersion, @@ -210,6 +307,21 @@ export class RoutingLambdaStack extends cdk.NestedStack { }) if (enableProvisionedConcurrency) { + const cachingTarget = new asg.ScalableTarget(this, 'CachingRoutingProvConcASG', { + serviceNamespace: asg.ServiceNamespace.LAMBDA, + maxCapacity: provisionedConcurrency * 10, + minCapacity: provisionedConcurrency, + resourceId: `function:${cachingRoutingLambdaAlias.lambda.functionName}:${cachingRoutingLambdaAlias.aliasName}`, + scalableDimension: 'lambda:function:ProvisionedConcurrency', + }) + + cachingTarget.node.addDependency(cachingRoutingLambdaAlias) + + cachingTarget.scaleToTrackMetric('CachingRoutingProvConcTracking', { + targetValue: 0.7, + predefinedMetric: asg.PredefinedMetric.LAMBDA_PROVISIONED_CONCURRENCY_UTILIZATION, + }) + const target = new asg.ScalableTarget(this, 'RoutingProvConcASG', { serviceNamespace: asg.ServiceNamespace.LAMBDA, maxCapacity: provisionedConcurrency * 10, diff --git a/lib/handlers/injector-sor.ts b/lib/handlers/injector-sor.ts index 9b8ae6e61..22357811f 100644 --- a/lib/handlers/injector-sor.ts +++ b/lib/handlers/injector-sor.ts @@ -184,6 +184,7 @@ export abstract class InjectorSOR extends Injector< CACHED_ROUTES_TABLE_NAME, AWS_LAMBDA_FUNCTION_NAME, V2_PAIRS_CACHE_TABLE_NAME, + CACHING_ROUTING_LAMBDA_FUNCTION_NAME, } = process.env const dependenciesByChain: { @@ -467,14 +468,26 @@ export abstract class InjectorSOR extends Injector< tenderlySimulator, ethEstimateGasSimulator ) + const newCachedRoutesRolloutPercent = NEW_CACHED_ROUTES_ROLLOUT_PERCENT[chainId] let routeCachingProvider: IRouteCachingProvider | undefined = undefined - if (CACHED_ROUTES_TABLE_NAME && CACHED_ROUTES_TABLE_NAME !== '') { - routeCachingProvider = new DynamoRouteCachingProvider({ - routesTableName: ROUTES_TABLE_NAME!, - routesCachingRequestFlagTableName: ROUTES_CACHING_REQUEST_FLAG_TABLE_NAME!, - cachingQuoteLambdaName: AWS_LAMBDA_FUNCTION_NAME!, - }) + + if (Math.random() * 100 < (newCachedRoutesRolloutPercent ?? 0)) { + if (CACHED_ROUTES_TABLE_NAME && CACHED_ROUTES_TABLE_NAME !== '') { + routeCachingProvider = new DynamoRouteCachingProvider({ + routesTableName: ROUTES_TABLE_NAME!, + routesCachingRequestFlagTableName: ROUTES_CACHING_REQUEST_FLAG_TABLE_NAME!, + cachingQuoteLambdaName: CACHING_ROUTING_LAMBDA_FUNCTION_NAME, + }) + } + } else { + if (CACHED_ROUTES_TABLE_NAME && CACHED_ROUTES_TABLE_NAME !== '') { + routeCachingProvider = new DynamoRouteCachingProvider({ + routesTableName: ROUTES_TABLE_NAME!, + routesCachingRequestFlagTableName: ROUTES_CACHING_REQUEST_FLAG_TABLE_NAME!, + cachingQuoteLambdaName: AWS_LAMBDA_FUNCTION_NAME!, + }) + } } const v2Supported = [ diff --git a/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts b/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts index 9ffb3d137..845e7785c 100644 --- a/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts +++ b/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts @@ -31,7 +31,7 @@ interface ConstructorParams { /** * The Lambda Function Name for the Lambda that will be invoked to fill the cache */ - cachingQuoteLambdaName: string + cachingQuoteLambdaName?: string } export class DynamoRouteCachingProvider extends IRouteCachingProvider { @@ -39,7 +39,7 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { private readonly lambdaClient: Lambda private readonly routesTableName: string private readonly routesCachingRequestFlagTableName: string - private readonly cachingQuoteLambdaName: string + private readonly cachingQuoteLambdaName?: string private readonly DEFAULT_CACHEMODE_ROUTES_DB = CacheMode.Livemode private readonly ROUTES_DB_TTL = 24 * 60 * 60 // 24 hours @@ -312,15 +312,17 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { }, } - const params = { - FunctionName: this.cachingQuoteLambdaName, - InvocationType: 'Event', - Payload: JSON.stringify(payload), - } + if (this.cachingQuoteLambdaName) { + const params = { + FunctionName: this.cachingQuoteLambdaName, + InvocationType: 'Event', + Payload: JSON.stringify(payload), + } - log.info(`[DynamoRouteCachingProvider] Sending async caching request to lambda ${JSON.stringify(params)}`) + log.info(`[DynamoRouteCachingProvider] Sending async caching request to lambda ${JSON.stringify(params)}`) - this.lambdaClient.invoke(params).promise() + this.lambdaClient.invoke(params).promise() + } } private setRoutesDbCachingIntentFlag(