Track V1 OApps latest configuration (#111)

packages/backend/.env.example

@@ -48,6 +48,15 @@ CELO_DISCOVERY_ENABLED=false
 CELO_RPC_URL=
 CELOSCAN_API_KEY=
 
+#-----TRACKING-----
+
+# Disables/enables oApps tracking on given chain
+ETHEREUM_TRACKING_ENABLED=false
+# RPC provider used to fetch default oApp configuration
+ETHEREUM_TRACKING_RPC_URL=
+# HTTP endpoint used to fetch data about availalbe oApps
+ETHEREUM_TRACKING_LIST_API_URL=
+
 #-----OPTIONAL-----
 
 # Define the data indexing root tick interval in milliseconds
@@ -146,4 +155,6 @@ CELO_START_BLOCK=
 CELO_RPC_LOGS_MAX_RANGE=
 CELO_EVENT_INDEXER_AMT_BATCHES=
 CELOSCAN_MIN_TIMESTAMP=
-CELO_LOGGER_ENABLED=
+CELO_LOGGER_ENABLED=
+
+

packages/backend/package.json

@@ -31,15 +31,16 @@
     "@l2beat/discovery-types": "0.8.0",
     "@l2beat/uif": "^0.2.3",
     "@lz/libs": "*",
+    "@lz/testnet": "*",
     "@sentry/node": "^7.73.0",
     "@types/deep-diff": "^1.0.4",
     "deep-diff": "^1.0.2",
-    "@lz/testnet": "*",
     "ethers": "^5.7.2",
     "knex": "^2.5.1",
     "koa": "^2.14.2",
     "koa-conditional-get": "^3.0.0",
     "koa-etag": "^4.0.0",
+    "node-fetch": "2",
     "pg": "^8.11.3",
     "source-map-support": "^0.5.21",
     "zod": "^3.22.2"

packages/backend/src/Application.ts

@@ -9,6 +9,7 @@ import { createHealthModule } from './modules/HealthModule'
 import { createStatusModule } from './modules/StatusModule'
 import { Database } from './peripherals/database/shared/Database'
 import { handleServerError, reportError } from './tools/ErrorReporter'
+import { createTrackingModule } from './tracking/TrackingModule'
 
 export class Application {
   start: () => Promise<void>
@@ -25,6 +26,7 @@ export class Application {
       createDiscoveryModule({ database, logger, config }),
       createStatusModule({ database, logger, config }),
       createConfigModule({ config }),
+      createTrackingModule({ database, logger, config }),
     ]
 
     const apiServer = new ApiServer(

packages/backend/src/config/Config.ts

@@ -39,6 +39,9 @@ export interface Config {
       readonly linea: DiscoverySubmoduleConfig
     }
   }
+  readonly tracking: {
+    readonly ethereum: TrackingSubmoduleConfig
+  }
 }
 
 export interface ApiConfig {
@@ -84,3 +87,21 @@ export interface EthereumLikeDiscoveryConfig {
   loggerEnabled?: boolean
   unsupportedEtherscanMethods?: EtherscanUnsupportedMethods
 }
+
+export type TrackingSubmoduleConfig =
+  | {
+      enabled: true
+      config: TrackingConfig
+    }
+  | {
+      enabled: false
+      config: null
+    }
+
+export interface TrackingConfig {
+  listApiUrl: string
+  rpcUrl: string
+  tickIntervalMs: number
+  multicall: MulticallConfig
+  ulnV2Address: EthereumAddress
+}

packages/backend/src/config/config.local.ts

@@ -1,7 +1,8 @@
 import { Env, LoggerOptions } from '@l2beat/backend-tools'
 
 import { Config } from './Config'
-import { getCommonDiscoveryConfig } from './config.common'
+import { getCommonDiscoveryConfig } from './config.discovery'
+import { getCommonTrackingConfig } from './config.tracking'
 import { getGitCommitSha } from './getGitCommitSha'
 
 export function getLocalConfig(env: Env): Config {
@@ -24,5 +25,6 @@ export function getLocalConfig(env: Env): Config {
       commitSha: getGitCommitSha(),
     },
     discovery: getCommonDiscoveryConfig(env),
+    tracking: getCommonTrackingConfig(env),
   }
 }

packages/backend/src/config/config.production.ts

@@ -1,7 +1,8 @@
 import { Env } from '@l2beat/backend-tools'
 
 import { Config } from './Config'
-import { getCommonDiscoveryConfig } from './config.common'
+import { getCommonDiscoveryConfig } from './config.discovery'
+import { getCommonTrackingConfig } from './config.tracking'
 import { getGitCommitSha } from './getGitCommitSha'
 
 export function getProductionConfig(env: Env): Config {
@@ -27,5 +28,6 @@ export function getProductionConfig(env: Env): Config {
       commitSha: getGitCommitSha(),
     },
     discovery: getCommonDiscoveryConfig(env),
+    tracking: getCommonTrackingConfig(env),
   }
 }

packages/backend/src/config/config.tracking.ts

@@ -0,0 +1,64 @@
+import { Env } from '@l2beat/backend-tools'
+import { MulticallConfig } from '@l2beat/discovery'
+import { EthereumAddress } from '@lz/libs'
+
+import { Config, TrackingSubmoduleConfig } from './Config'
+import { coreAddressesV1, ethereumMulticallConfig } from './discovery/ethereum'
+
+export function getCommonTrackingConfig(env: Env): Config['tracking'] {
+  const createTrackingConfig = configFromTemplate(env)
+
+  return {
+    ethereum: createTrackingConfig({
+      chainNamePrefix: 'ETHEREUM',
+      multicallConfig: ethereumMulticallConfig,
+      ulnV2Address: EthereumAddress(coreAddressesV1.ultraLightNodeV2),
+    }),
+  }
+}
+
+function configFromTemplate(env: Env) {
+  return function ({
+    chainNamePrefix,
+    multicallConfig,
+    ulnV2Address,
+  }: {
+    /**
+     * The prefix of the environment variables that configure the chain.
+     */
+    chainNamePrefix: string
+
+    /**
+     * Multicall configuration for given chain
+     */
+    multicallConfig: MulticallConfig
+
+    /**
+     * Address of UlnV2 on given chain
+     */
+    ulnV2Address: EthereumAddress
+  }): TrackingSubmoduleConfig {
+    const isEnabled = env.boolean(`${chainNamePrefix}_TRACKING_ENABLED`, false)
+
+    if (!isEnabled) {
+      return {
+        enabled: false,
+        config: null,
+      }
+    }
+
+    return {
+      enabled: true,
+      config: {
+        listApiUrl: env.string(
+          `${chainNamePrefix}_TRACKING_LIST_API_URL`,
+          'https://l2beat-production.herokuapp.com/api/integrations/layerzero-oapps',
+        ),
+        tickIntervalMs: env.integer('TRACKING_TICK_INTERVAL_MS', 60_000_000),
+        rpcUrl: env.string(`${chainNamePrefix}_TRACKING_RPC_URL`),
+        ulnV2Address: ulnV2Address,
+        multicall: multicallConfig,
+      },
+    }
+  }
+}

packages/backend/src/config/discovery/ethereum.ts

@@ -14,6 +14,7 @@ import {
 } from '../types'
 
 export {
+  coreAddressesV1,
   ethereumChangelogWhitelist,
   ethereumDiscoveryConfig,
   ethereumEventsToWatch,

packages/backend/src/modules/DiscoveryModule.ts

@@ -137,9 +137,7 @@ export function createDiscoveryModule({
     start: async () => {
       statusLogger.info('Starting discovery module')
 
-      for (const submodule of submodules) {
-        await submodule.start?.()
-      }
+      await Promise.all(submodules.map((submodule) => submodule.start?.()))
 
       statusLogger.info('Main discovery module started')
     },

packages/backend/src/peripherals/database/OAppConfigurationRepository.test.ts

@@ -0,0 +1,73 @@
+import { Logger } from '@l2beat/backend-tools'
+import { ChainId, EthereumAddress } from '@lz/libs'
+import { expect } from 'earl'
+
+import { setupDatabaseTestSuite } from '../../test/database'
+import {
+  OAppConfigurationRecord,
+  OAppConfigurationRepository,
+} from './OAppConfigurationRepository'
+
+describe(OAppConfigurationRepository.name, () => {
+  const { database } = setupDatabaseTestSuite()
+  const repository = new OAppConfigurationRepository(database, Logger.SILENT)
+
+  before(async () => await repository.deleteAll())
+  afterEach(async () => await repository.deleteAll())
+
+  describe(OAppConfigurationRepository.prototype.addMany.name, () => {
+    it('merges rows on insert', async () => {
+      const record1 = mockRecord({ oAppId: 1, targetChainId: ChainId.ETHEREUM })
+      const record2 = mockRecord({ oAppId: 2, targetChainId: ChainId.OPTIMISM })
+
+      await repository.addMany([record1, record2])
+
+      const recordsBeforeMerge = await repository.findAll()
+
+      await repository.addMany([record1, record2])
+
+      const recordsAfterMerge = await repository.findAll()
+
+      expect(recordsBeforeMerge.length).toEqual(2)
+      expect(recordsAfterMerge.length).toEqual(2)
+    })
+  })
+
+  describe(OAppConfigurationRepository.prototype.findByOAppIds.name, () => {
+    it('returns only records with matching oAppId', async () => {
+      const record1 = mockRecord({
+        oAppId: 1,
+      })
+      const record2 = mockRecord({
+        oAppId: 2,
+      })
+      const record3 = mockRecord({
+        oAppId: 3,
+      })
+
+      await repository.addMany([record1, record2, record3])
+
+      const result = await repository.findByOAppIds([1, 2])
+
+      expect(result.length).toEqual(2)
+    })
+  })
+})
+
+function mockRecord(
+  overrides?: Partial<OAppConfigurationRecord>,
+): OAppConfigurationRecord {
+  return {
+    oAppId: 1,
+    targetChainId: ChainId.ETHEREUM,
+    configuration: {
+      inboundBlockConfirmations: 2,
+      outboundBlockConfirmations: 2,
+      relayer: EthereumAddress.random(),
+      oracle: EthereumAddress.random(),
+      outboundProofType: 2,
+      inboundProofLibraryVersion: 2,
+    },
+    ...overrides,
+  }
+}

packages/backend/src/peripherals/database/OAppConfigurationRepository.ts

@@ -0,0 +1,72 @@
+import { Logger } from '@l2beat/backend-tools'
+import { ChainId } from '@lz/libs'
+import type { OAppConfigurationRow } from 'knex/types/tables'
+
+import { OAppConfiguration } from '../../tracking/domain/configuration'
+import { BaseRepository, CheckConvention } from './shared/BaseRepository'
+import { Database } from './shared/Database'
+
+export interface OAppConfigurationRecord {
+  oAppId: number
+  targetChainId: ChainId
+  configuration: OAppConfiguration
+}
+
+export class OAppConfigurationRepository extends BaseRepository {
+  constructor(database: Database, logger: Logger) {
+    super(database, logger)
+    this.autoWrap<CheckConvention<OAppConfigurationRepository>>(this)
+  }
+
+  public async addMany(records: OAppConfigurationRecord[]): Promise<number> {
+    const rows = records.map(toRow)
+    const knex = await this.knex()
+
+    await knex('oapp_configuration')
+      .insert(rows)
+      .onConflict(['oapp_id', 'target_chain_id'])
+      .merge()
+
+    return rows.length
+  }
+
+  public async findAll(): Promise<OAppConfigurationRecord[]> {
+    const knex = await this.knex()
+
+    const rows = await knex('oapp_configuration').select('*')
+
+    return rows.map(toRecord)
+  }
+  public async findByOAppIds(
+    oAppIds: number[],
+  ): Promise<OAppConfigurationRecord[]> {
+    const knex = await this.knex()
+
+    const rows = await knex('oapp_configuration')
+      .select('*')
+      .whereIn('oapp_id', oAppIds)
+
+    return rows.map(toRecord)
+  }
+
+  async deleteAll(): Promise<number> {
+    const knex = await this.knex()
+    return knex('oapp_configuration').delete()
+  }
+}
+
+function toRow(record: OAppConfigurationRecord): OAppConfigurationRow {
+  return {
+    oapp_id: record.oAppId,
+    target_chain_id: Number(record.targetChainId),
+    configuration: JSON.stringify(record.configuration),
+  }
+}
+
+function toRecord(row: OAppConfigurationRow): OAppConfigurationRecord {
+  return {
+    oAppId: row.oapp_id,
+    targetChainId: ChainId(row.target_chain_id),
+    configuration: OAppConfiguration.parse(row.configuration),
+  }
+}