From 1f993892cd77617a0edfc716c0dc018dbcd2c770 Mon Sep 17 00:00:00 2001 From: "Siyu Jiang (See-You John)" <91580504+jsy1218@users.noreply.github.com> Date: Tue, 29 Oct 2024 06:29:18 -0700 Subject: [PATCH] chore: bump sor to 4.7.5 - fix: cached routes cache invalidation (#869) * sor 4.7.1 from local * fix: we roll out new cached routes fix with percent * fix: we roll out new cached routes fix with percent * add more comments on per-chain rollout percent * fix injector.ts to pass in alpha-router params * make BLOCKS_TO_LIVE to index key on ChainId enum * remove filter expired cached routes * 10 minutes blocks-to-live for every chain * INTENT enum * npm install SOR * sdk-core 5.9.0 * fix cachedRoutesCacheInvalidationFixRolloutPercentage * fix v4PoolParams * do not tap compare during caching intent quote handler * 1 hour cached routes refresh --- lib/handlers/injector-sor.ts | 14 ++++- lib/handlers/quote/injector.ts | 8 +++ .../dynamo-route-caching-provider.ts | 55 +++---------------- lib/handlers/shared.ts | 8 ++- lib/util/defaultBlocksToLiveRoutesDB.ts | 48 ++++++++++++++++ lib/util/newCachedRoutesRolloutPercent.ts | 34 ++++++++++++ package-lock.json | 32 +++++------ package.json | 4 +- 8 files changed, 134 insertions(+), 69 deletions(-) create mode 100644 lib/util/defaultBlocksToLiveRoutesDB.ts create mode 100644 lib/util/newCachedRoutesRolloutPercent.ts diff --git a/lib/handlers/injector-sor.ts b/lib/handlers/injector-sor.ts index 8a76fd56c..9b8ae6e61 100644 --- a/lib/handlers/injector-sor.ts +++ b/lib/handlers/injector-sor.ts @@ -90,6 +90,7 @@ import { emptyV4FeeTickSpacingsHookAddresses, EXTRA_V4_FEE_TICK_SPACINGS_HOOK_ADDRESSES, } from '../util/extraV4FeeTiersTickSpacingsHookAddresses' +import { NEW_CACHED_ROUTES_ROLLOUT_PERCENT } from '../util/newCachedRoutesRolloutPercent' export const SUPPORTED_CHAINS: ChainId[] = [ ChainId.MAINNET, @@ -144,6 +145,10 @@ export type ContainerDependencies = { tokenValidatorProvider: TokenValidatorProvider tokenPropertiesProvider: ITokenPropertiesProvider v2Supported: ChainId[] + v4Supported?: ChainId[] + mixedSupported?: ChainId[] + v4PoolParams?: Array<[number, number, string]> + cachedRoutesCacheInvalidationFixRolloutPercentage?: number } export interface ContainerInjected { @@ -297,7 +302,7 @@ export abstract class InjectorSOR extends Injector< underlyingV2PoolProvider, new V2DynamoCache(V2_PAIRS_CACHE_TABLE_NAME!) ) - const v4PoolsParams = getApplicableV4FeesTickspacingsHooks(chainId).concat( + const v4PoolParams = getApplicableV4FeesTickspacingsHooks(chainId).concat( EXTRA_V4_FEE_TICK_SPACINGS_HOOK_ADDRESSES[chainId] ?? emptyV4FeeTickSpacingsHookAddresses ) @@ -316,7 +321,7 @@ export abstract class InjectorSOR extends Injector< POOL_CACHE_BUCKET_3!, POOL_CACHE_GZIP_KEY!, v4PoolProvider, - v4PoolsParams + v4PoolParams )) as V4AWSSubgraphProvider, (await this.instantiateSubgraphProvider( chainId, @@ -487,6 +492,8 @@ export abstract class InjectorSOR extends Injector< const mixedSupported = [ChainId.MAINNET, ChainId.SEPOLIA, ChainId.GOERLI] + const cachedRoutesCacheInvalidationFixRolloutPercentage = NEW_CACHED_ROUTES_ROLLOUT_PERCENT[chainId] + return { chainId, dependencies: { @@ -520,7 +527,8 @@ export abstract class InjectorSOR extends Injector< v2Supported, v4Supported, mixedSupported, - v4PoolsParams, + v4PoolParams, + cachedRoutesCacheInvalidationFixRolloutPercentage, }, } }) diff --git a/lib/handlers/quote/injector.ts b/lib/handlers/quote/injector.ts index 1ba5c6374..86ec67dbb 100644 --- a/lib/handlers/quote/injector.ts +++ b/lib/handlers/quote/injector.ts @@ -101,6 +101,10 @@ export class QuoteHandlerInjector extends InjectorSOR< simulator, routeCachingProvider, v2Supported, + v4Supported, + mixedSupported, + v4PoolParams, + cachedRoutesCacheInvalidationFixRolloutPercentage, } = dependencies[chainIdEnum]! let onChainQuoteProvider = dependencies[chainIdEnum]!.onChainQuoteProvider @@ -135,6 +139,10 @@ export class QuoteHandlerInjector extends InjectorSOR< tokenValidatorProvider, tokenPropertiesProvider, v2Supported, + v4Supported, + mixedSupported, + v4PoolParams, + cachedRoutesCacheInvalidationFixRolloutPercentage, }) break } diff --git a/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts b/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts index 7eca9af85..9ffb3d137 100644 --- a/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts +++ b/lib/handlers/router-entities/route-caching/dynamo-route-caching-provider.ts @@ -3,6 +3,7 @@ import { CachedRoutes, CacheMode, ID_TO_NETWORK_NAME, + INTENT, IRouteCachingProvider, log, metric, @@ -16,6 +17,7 @@ import { Protocol } from '@uniswap/router-sdk' import { PairTradeTypeChainId } from './model/pair-trade-type-chain-id' import { CachedRoutesMarshaller } from '../../marshalling/cached-routes-marshaller' import { PromiseResult } from 'aws-sdk/lib/request' +import { DEFAULT_BLOCKS_TO_LIVE_ROUTES_DB } from '../../../util/defaultBlocksToLiveRoutesDB' interface ConstructorParams { /** @@ -43,43 +45,6 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { private readonly ROUTES_DB_TTL = 24 * 60 * 60 // 24 hours private readonly ROUTES_DB_FLAG_TTL = 2 * 60 // 2 minutes - // heuristic is within 30 seconds we find a route. - // we know each chain block time - // divide those two - private readonly DEFAULT_BLOCKS_TO_LIVE_ROUTES_DB = (chainId: ChainId) => { - switch (chainId) { - // https://dune.com/queries/2138021 - case ChainId.ARBITRUM_ONE: - return 100 - - // https://dune.com/queries/2009572 - case ChainId.BASE: - case ChainId.OPTIMISM: - return 60 - - // https://snowtrace.io/chart/blocktime - case ChainId.AVALANCHE: - return 15 - - // https://dune.com/KARTOD/blockchains-analysis - case ChainId.BNB: - return 10 - - // https://dune.com/KARTOD/blockchains-analysis - case ChainId.POLYGON: - return 15 - - // https://explorer.celo.org/mainnet/ - case ChainId.CELO: - return 6 - - // https://dune.com/KARTOD/blockchains-analysis - case ChainId.MAINNET: - default: - return 2 - } - } - // For the Ratio we are approximating Phi (Golden Ratio) by creating a fraction with 2 consecutive Fibonacci numbers private readonly ROUTES_DB_BUCKET_RATIO: Fraction = new Fraction(514229, 317811) private readonly ROUTES_TO_TAKE_FROM_ROUTES_DB = 8 private readonly BLOCKS_DIFF_BETWEEN_CACHING_QUOTES: Map = new Map([[ChainId.MAINNET, 3]]) @@ -114,7 +79,7 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { * @protected */ protected async _getBlocksToLive(cachedRoutes: CachedRoutes, _: CurrencyAmount): Promise { - return this.DEFAULT_BLOCKS_TO_LIVE_ROUTES_DB(cachedRoutes.chainId) + return DEFAULT_BLOCKS_TO_LIVE_ROUTES_DB[cachedRoutes.chainId] } /** @@ -342,7 +307,7 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { amount: amount.quotient.toString(), type: partitionKey.tradeType === 0 ? 'exactIn' : 'exactOut', protocols: protocols.map((protocol) => protocol.toLowerCase()).join(','), - intent: 'caching', + intent: INTENT.CACHING, requestSource: 'routing-api', }, } @@ -475,14 +440,10 @@ export class DynamoRouteCachingProvider extends IRouteCachingProvider { _blockNumber: number, _optimistic: boolean ): CachedRoutes | undefined { - // if it's on sepolia, then we want to filter expired routes by blocks to live - // this is to unblock v4 routing tests on sepolia - if (cachedRoutes?.chainId === ChainId.SEPOLIA) { - return cachedRoutes?.notExpired(_blockNumber, _optimistic) ? cachedRoutes : undefined - } else { - // otherwise, we keep it here, but we need a better plan for how to fix filtering expired cached routes - return cachedRoutes - } + // we can revert it back to never filter expired cached routes in read path + // we will use blocks-to-live in the write cached routes path + // also we can verify that cached routes cache invalidation bug error is gone in sepolia, as we removed hardcoding for sepolia + return cachedRoutes } /** diff --git a/lib/handlers/shared.ts b/lib/handlers/shared.ts index e1f016f3a..68f99243b 100644 --- a/lib/handlers/shared.ts +++ b/lib/handlers/shared.ts @@ -2,6 +2,7 @@ import { ChainId, Currency, CurrencyAmount, Percent } from '@uniswap/sdk-core' import { AlphaRouterConfig, CacheMode, + INTENT, LowerCaseStringArray, MapWithLowerCaseKey, ProtocolPoolSelection, @@ -238,6 +239,7 @@ export const QUOTE_SPEED_CONFIG: { [key: string]: QuoteSpeedConfig } = { } export type IntentSpecificConfig = { + intent?: INTENT useCachedRoutes?: boolean overwriteCacheMode?: CacheMode optimisticCachedRoutes?: boolean @@ -246,23 +248,27 @@ export type IntentSpecificConfig = { export const INTENT_SPECIFIC_CONFIG: { [key: string]: IntentSpecificConfig } = { caching: { // When the intent is to create a cache entry, we will use cachedRoutes with Tapcompare to track accuracy + intent: INTENT.CACHING, useCachedRoutes: true, - overwriteCacheMode: CacheMode.Tapcompare, + // overwriteCacheMode: CacheMode.Tapcompare, // This optimistic=false is *super* important to avoid an infinite loop of caching quotes calling themselves optimisticCachedRoutes: false, }, quote: { // When the intent is to get a quote, we should use the cache and optimistic cached routes + intent: INTENT.QUOTE, useCachedRoutes: true, optimisticCachedRoutes: true, }, swap: { // When the intent is to prepare the swap, we can use cache, but it should not be optimistic + intent: INTENT.SWAP, useCachedRoutes: true, optimisticCachedRoutes: false, }, pricing: { // When the intent is to get pricing, we should use the cache and optimistic cached routes + intent: INTENT.PRICING, useCachedRoutes: true, optimisticCachedRoutes: true, }, diff --git a/lib/util/defaultBlocksToLiveRoutesDB.ts b/lib/util/defaultBlocksToLiveRoutesDB.ts new file mode 100644 index 000000000..56f811fea --- /dev/null +++ b/lib/util/defaultBlocksToLiveRoutesDB.ts @@ -0,0 +1,48 @@ +import { ChainId } from '@uniswap/sdk-core' + +// This is not being used in production today anyway, due to below filterExpiredCachedRoutes method not really filtering on the blocks-to-live +// heuristic is initially within 30 seconds we find a route. (but we are changing to every 1 hour now) +// For the Ratio we are approximating Phi (Golden Ratio) by creating a fraction with 2 consecutive Fibonacci numbers + +// changing to this way with ChainId enum as key indexing, so that we wont forgot to add new chain tuned blocks-to-live +// those are only gonna be enabled with DynamoRouteCachingProvider.newCachedRoutesRolloutPercent anyway +export const DEFAULT_BLOCKS_TO_LIVE_ROUTES_DB: { [chain in ChainId]: number } = { + // (60 minutes) / (12 seconds)= 300 + [ChainId.MAINNET]: 300, + [ChainId.GOERLI]: 300, + [ChainId.SEPOLIA]: 300, + // (60 minutes) / (2 seconds) = 1800 + [ChainId.OPTIMISM]: 1800, + [ChainId.OPTIMISM_GOERLI]: 1800, + [ChainId.OPTIMISM_SEPOLIA]: 1800, + [ChainId.BASE]: 1800, + [ChainId.ZORA]: 1800, + [ChainId.BASE_GOERLI]: 1800, + [ChainId.ZORA_SEPOLIA]: 1800, + [ChainId.BLAST]: 1800, + [ChainId.WORLDCHAIN]: 1800, + // (60 minutes) / (1 seconds) = 3600 + [ChainId.ASTROCHAIN_SEPOLIA]: 3600, + // (60 minutes) / (250 milliseconds) = 14400 + [ChainId.ARBITRUM_ONE]: 14400, + [ChainId.ARBITRUM_GOERLI]: 14400, + [ChainId.ARBITRUM_SEPOLIA]: 14400, + // (60 minutes) / (2 seconds) = 1800 + [ChainId.POLYGON]: 1800, + [ChainId.POLYGON_MUMBAI]: 1800, + // (60 minutes) / (5 seconds) = 720 + [ChainId.CELO]: 720, + [ChainId.CELO_ALFAJORES]: 720, + // (60 minutes) / (5 seconds) = 720 + [ChainId.GNOSIS]: 720, + // (60 minutes) / (6 seconds) = 600 + [ChainId.MOONBEAM]: 600, + // (60 minutes) / (3 seconds) = 1200 + [ChainId.BNB]: 1200, + // (60 minutes) / (3 seconds) = 1200 + [ChainId.AVALANCHE]: 1200, + // (60 minutes) / (33 seconds) = 148 + [ChainId.ROOTSTOCK]: 148, + // (60 minutes) / (1 seconds) = 3600 + [ChainId.ZKSYNC]: 3600, +} diff --git a/lib/util/newCachedRoutesRolloutPercent.ts b/lib/util/newCachedRoutesRolloutPercent.ts new file mode 100644 index 000000000..16400e172 --- /dev/null +++ b/lib/util/newCachedRoutesRolloutPercent.ts @@ -0,0 +1,34 @@ +import { ChainId } from '@uniswap/sdk-core' + +// percent is between 0% - 100%, defined in SOR +// testnets all go to 100% directly +// production nets depending on revenue/quote traffic volume, if it's medium/high 1%, otherwise super low traffic (< 100 quotes per 5 minutes) 100% +// so zora and blast go to 100% directly. rootstock is not supported by uniswap labs product or protocol layer, go to 100% directly. +export const NEW_CACHED_ROUTES_ROLLOUT_PERCENT: { [chain in ChainId]: number } = { + [ChainId.MAINNET]: 1, + [ChainId.GOERLI]: 100, + [ChainId.SEPOLIA]: 100, + [ChainId.OPTIMISM]: 1, + [ChainId.OPTIMISM_GOERLI]: 100, + [ChainId.OPTIMISM_SEPOLIA]: 100, + [ChainId.ARBITRUM_ONE]: 1, + [ChainId.ARBITRUM_GOERLI]: 100, + [ChainId.ARBITRUM_SEPOLIA]: 100, + [ChainId.POLYGON]: 1, + [ChainId.POLYGON_MUMBAI]: 100, + [ChainId.CELO]: 1, + [ChainId.CELO_ALFAJORES]: 100, + [ChainId.GNOSIS]: 1, + [ChainId.MOONBEAM]: 1, + [ChainId.BNB]: 1, + [ChainId.AVALANCHE]: 1, + [ChainId.BASE_GOERLI]: 100, + [ChainId.BASE]: 1, + [ChainId.ZORA]: 100, + [ChainId.ZORA_SEPOLIA]: 100, + [ChainId.ROOTSTOCK]: 100, + [ChainId.BLAST]: 100, + [ChainId.ZKSYNC]: 1, + [ChainId.WORLDCHAIN]: 1, + [ChainId.ASTROCHAIN_SEPOLIA]: 100, +} diff --git a/package-lock.json b/package-lock.json index f29a38677..f2474c732 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,8 +27,8 @@ "@uniswap/default-token-list": "^11.13.0", "@uniswap/permit2-sdk": "^1.3.0", "@uniswap/router-sdk": "^1.14.0", - "@uniswap/sdk-core": "^5.8.5", - "@uniswap/smart-order-router": "4.7.3", + "@uniswap/sdk-core": "^5.9.0", + "@uniswap/smart-order-router": "4.7.5", "@uniswap/token-lists": "^1.0.0-beta.33", "@uniswap/universal-router-sdk": "^4.6.1", "@uniswap/v2-sdk": "^4.6.1", @@ -4731,9 +4731,9 @@ } }, "node_modules/@uniswap/sdk-core": { - "version": "5.8.5", - "resolved": "https://registry.npmjs.org/@uniswap/sdk-core/-/sdk-core-5.8.5.tgz", - "integrity": "sha512-eLsBnN87VvxjhATp96h6kB4vIdxxOQxTH6bsD9q/flSLpLrqhuJAisAhYCf/sCMzXGpZbxLqlgmGn9wueQIkXg==", + "version": "5.9.0", + "resolved": "https://registry.npmjs.org/@uniswap/sdk-core/-/sdk-core-5.9.0.tgz", + "integrity": "sha512-OME7WR6+5QwQs45A2079r+/FS0zU944+JCQwUX9GyIriCxqw2pGu4F9IEqmlwD+zSIMml0+MJnJJ47pFgSyWDw==", "dependencies": { "@ethersproject/address": "^5.0.2", "@ethersproject/bytes": "^5.7.0", @@ -4750,16 +4750,16 @@ } }, "node_modules/@uniswap/smart-order-router": { - "version": "4.7.3", - "resolved": "https://registry.npmjs.org/@uniswap/smart-order-router/-/smart-order-router-4.7.3.tgz", - "integrity": "sha512-ehUvGt9Htn2qaHIfFcEFSPQxp29RAgBqV5ODMeqMebJGGeaw0uPS/ovR1gnzBNFC8FHKDCoiRFSHZ0M6U2nPaQ==", + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/@uniswap/smart-order-router/-/smart-order-router-4.7.5.tgz", + "integrity": "sha512-3BlkCn0cPWhuWD1ACuIAmYJ5PAwhDYfvJloYAzREZj9/Ie5cpwUhXtHD3hBrS97S4Cbyh2UJ57dpwKpl8p/efA==", "dependencies": { "@eth-optimism/sdk": "^3.2.2", "@types/brotli": "^1.3.4", "@uniswap/default-token-list": "^11.13.0", "@uniswap/permit2-sdk": "^1.3.0", "@uniswap/router-sdk": "^1.14.0", - "@uniswap/sdk-core": "^5.8.5", + "@uniswap/sdk-core": "^5.9.0", "@uniswap/swap-router-contracts": "^1.3.1", "@uniswap/token-lists": "^1.0.0-beta.31", "@uniswap/universal-router": "^1.6.0", @@ -28433,9 +28433,9 @@ } }, "@uniswap/sdk-core": { - "version": "5.8.5", - "resolved": "https://registry.npmjs.org/@uniswap/sdk-core/-/sdk-core-5.8.5.tgz", - "integrity": "sha512-eLsBnN87VvxjhATp96h6kB4vIdxxOQxTH6bsD9q/flSLpLrqhuJAisAhYCf/sCMzXGpZbxLqlgmGn9wueQIkXg==", + "version": "5.9.0", + "resolved": "https://registry.npmjs.org/@uniswap/sdk-core/-/sdk-core-5.9.0.tgz", + "integrity": "sha512-OME7WR6+5QwQs45A2079r+/FS0zU944+JCQwUX9GyIriCxqw2pGu4F9IEqmlwD+zSIMml0+MJnJJ47pFgSyWDw==", "requires": { "@ethersproject/address": "^5.0.2", "@ethersproject/bytes": "^5.7.0", @@ -28449,16 +28449,16 @@ } }, "@uniswap/smart-order-router": { - "version": "4.7.3", - "resolved": "https://registry.npmjs.org/@uniswap/smart-order-router/-/smart-order-router-4.7.3.tgz", - "integrity": "sha512-ehUvGt9Htn2qaHIfFcEFSPQxp29RAgBqV5ODMeqMebJGGeaw0uPS/ovR1gnzBNFC8FHKDCoiRFSHZ0M6U2nPaQ==", + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/@uniswap/smart-order-router/-/smart-order-router-4.7.5.tgz", + "integrity": "sha512-3BlkCn0cPWhuWD1ACuIAmYJ5PAwhDYfvJloYAzREZj9/Ie5cpwUhXtHD3hBrS97S4Cbyh2UJ57dpwKpl8p/efA==", "requires": { "@eth-optimism/sdk": "^3.2.2", "@types/brotli": "^1.3.4", "@uniswap/default-token-list": "^11.13.0", "@uniswap/permit2-sdk": "^1.3.0", "@uniswap/router-sdk": "^1.14.0", - "@uniswap/sdk-core": "^5.8.5", + "@uniswap/sdk-core": "^5.9.0", "@uniswap/swap-router-contracts": "^1.3.1", "@uniswap/token-lists": "^1.0.0-beta.31", "@uniswap/universal-router": "^1.6.0", diff --git a/package.json b/package.json index bc8f4133f..5c8580c7e 100644 --- a/package.json +++ b/package.json @@ -85,9 +85,9 @@ "@uniswap/default-token-list": "^11.13.0", "@uniswap/permit2-sdk": "^1.3.0", "@uniswap/router-sdk": "^1.14.0", - "@uniswap/sdk-core": "^5.8.5", + "@uniswap/sdk-core": "^5.9.0", "@types/semver": "^7.5.8", - "@uniswap/smart-order-router": "4.7.3", + "@uniswap/smart-order-router": "4.7.5", "@uniswap/token-lists": "^1.0.0-beta.33", "@uniswap/universal-router-sdk": "^4.6.1", "@uniswap/v2-sdk": "^4.6.1",