Merge pull request #30 from lidofinance/develop

Develop

Dockerfile

@@ -23,13 +23,17 @@ RUN yarn build
 
 FROM node:14.18.1-alpine3.13
 
+ENV PORT=
+
 RUN mkdir /council
 
 WORKDIR /council
 
 COPY --from=building /council/dist ./dist
 COPY --from=building /council/node_modules ./node_modules
 COPY ./package*.json ./
-COPY ./yarn*.lock ./
+
+HEALTHCHECK --interval=120s --timeout=2s --retries=2 \
+    CMD sh -c "wget -nv -t1 --spider http://localhost:$PORT/health" || exit 1
 
 CMD ["yarn", "start:prod"]

package.json

@@ -1,6 +1,6 @@
 {
   "name": "lido-council-daemon",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "Lido Council Daemon",
   "author": "Lido team",
   "private": true,
@@ -35,6 +35,7 @@
     "@nestjs/config": "^1.0.1",
     "@nestjs/core": "^8.0.0",
     "@nestjs/platform-express": "^8.1.1",
+    "@nestjs/terminus": "^8.0.1",
     "@willsoto/nestjs-prometheus": "^4.0.1",
     "app-root-path": "^3.0.0",
     "class-transformer": "^0.4.0",

src/app.module.ts

@@ -6,6 +6,7 @@ import { LoggerModule } from 'common/logger';
 import { PrometheusModule } from 'common/prometheus';
 import { GuardianModule } from 'guardian';
 import { ProviderModule } from 'provider';
+import { HealthModule } from 'health';
 
 @Module({
   imports: [
@@ -14,6 +15,7 @@ import { ProviderModule } from 'provider';
     PrometheusModule,
     LoggerModule,
     GuardianModule,
+    HealthModule,
   ],
   providers: [AppService],
 })

src/contracts/deposit/deposit.service.ts

@@ -42,35 +42,81 @@ export class DepositService {
   }
 
   async onModuleInit() {
-    const cache = await this.getCachedEvents();
-    const versions = {
-      cachedVersion: cache.version,
-      currentVersion: APP_VERSION,
-    };
-
-    if (cache.version === APP_VERSION) {
-      this.logger.log(
-        'Deposit Events cache version matches the application version',
-        versions,
-      );
-
-      return;
-    }
+    const currentBlock = await this.providerService.getBlockNumber();
+    const cachedEvents = await this.getCachedEvents();
+    const isCacheValid = this.validateCache(cachedEvents, currentBlock);
 
-    this.logger.log(
-      'Deposit Events cache does not match the application version, clearing the cache',
-      versions,
-    );
+    if (isCacheValid) return;
 
     try {
       await this.deleteCachedEvents();
-      this.logger.log('Deposit Events cache cleared');
+      this.logger.warn('Deposit events cache cleared');
     } catch (error) {
       this.logger.error(error);
       process.exit(1);
     }
   }
 
+  /**
+   * Validates the app cache
+   * @param cachedEvents - cached events
+   * @param currentBlock - current block number
+   * @returns true if cache is valid
+   */
+  public validateCache(
+    cachedEvents: DepositEventsCache,
+    currentBlock: number,
+  ): boolean {
+    return (
+      this.validateCacheBlock(cachedEvents, currentBlock) &&
+      this.validateCacheVersion(cachedEvents)
+    );
+  }
+
+  /**
+   * Validates app version in the cache
+   * @param cachedEvents - cached events
+   * @returns true if cached app version is the same
+   */
+  public validateCacheVersion(cachedEvents: DepositEventsCache): boolean {
+    const isSameVersion = cachedEvents.version === APP_VERSION;
+
+    if (!isSameVersion) {
+      this.logger.warn(
+        'Deposit events cache does not match the application version, clearing the cache',
+        {
+          cachedVersion: cachedEvents.version,
+          currentVersion: APP_VERSION,
+        },
+      );
+    }
+
+    return isSameVersion;
+  }
+
+  /**
+   * Validates block number in the cache
+   * @param cachedEvents - cached events
+   * @param currentBlock - current block number
+   * @returns true if cached app version is the same
+   */
+  public validateCacheBlock(
+    cachedEvents: DepositEventsCache,
+    currentBlock: number,
+  ): boolean {
+    const isCacheValid = currentBlock >= cachedEvents.endBlock;
+
+    if (!isCacheValid) {
+      this.logger.warn('Deposit events cache is newer than the current block', {
+        cachedStartBlock: cachedEvents.startBlock,
+        cachedEndBlock: cachedEvents.endBlock,
+        currentBlock,
+      });
+    }
+
+    return isCacheValid;
+  }
+
   /**
    * Returns only required information about the event,
    * to reduce the size of the information stored in the cache
@@ -265,6 +311,9 @@ export class DepositService {
     const endBlock = blockNumber;
     const cachedEvents = await this.getCachedEvents();
 
+    const isCacheValid = this.validateCacheBlock(cachedEvents, blockNumber);
+    if (!isCacheValid) process.exit(1);
+
     const firstNotCachedBlock = cachedEvents.endBlock + 1;
     const freshEventGroup = await this.fetchEventsFallOver(
       firstNotCachedBlock,
@@ -292,7 +341,7 @@ export class DepositService {
     events: DepositEvent[],
     blockNumber: number,
     blockHash: string,
-  ) {
+  ): void {
     events.forEach((event) => {
       if (event.blockNumber === blockNumber && event.blockHash !== blockHash) {
         throw new Error(

src/health/health.constants.ts

@@ -0,0 +1 @@
+export const MAX_BLOCK_DELAY_SECONDS = 5 * 60;

src/health/health.controller.ts

@@ -0,0 +1,25 @@
+import { Controller, Get } from '@nestjs/common';
+import {
+  HealthCheckService,
+  MemoryHealthIndicator,
+  HealthCheck,
+} from '@nestjs/terminus';
+import { ProviderHealthIndicator } from './provider.health';
+
+@Controller('health')
+export class HealthController {
+  constructor(
+    private health: HealthCheckService,
+    private memory: MemoryHealthIndicator,
+    private provider: ProviderHealthIndicator,
+  ) {}
+
+  @Get()
+  @HealthCheck()
+  check() {
+    return this.health.check([
+      async () => this.memory.checkHeap('memoryHeap', 1024 * 1024 * 1024),
+      async () => this.provider.isHealthy('RPCProvider'),
+    ]);
+  }
+}

src/health/health.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { TerminusModule } from '@nestjs/terminus';
+import { HealthController } from './health.controller';
+import { ProviderHealthIndicator } from './provider.health';
+
+@Module({
+  providers: [ProviderHealthIndicator],
+  controllers: [HealthController],
+  imports: [TerminusModule],
+})
+export class HealthModule {}

src/health/index.ts

@@ -0,0 +1,2 @@
+export * from './health.controller';
+export * from './health.module';

src/health/provider.health.ts

@@ -0,0 +1,43 @@
+import { Injectable } from '@nestjs/common';
+import {
+  HealthIndicator,
+  HealthIndicatorResult,
+  HealthCheckError,
+} from '@nestjs/terminus';
+import { ProviderService } from 'provider';
+import { MAX_BLOCK_DELAY_SECONDS } from './health.constants';
+
+@Injectable()
+export class ProviderHealthIndicator extends HealthIndicator {
+  constructor(private providerService: ProviderService) {
+    super();
+  }
+
+  async getBlockTimestamp() {
+    try {
+      const block = await this.providerService.getBlock();
+      return block.timestamp;
+    } catch (error) {
+      return -1;
+    }
+  }
+
+  getNowTimestamp() {
+    return Math.floor(Date.now() / 1000);
+  }
+
+  async isHealthy(key: string): Promise<HealthIndicatorResult> {
+    const blockTimestamp = await this.getBlockTimestamp();
+    const nowTimestamp = this.getNowTimestamp();
+    const deltaTimestamp = Math.abs(nowTimestamp - blockTimestamp);
+
+    const isHealthy = deltaTimestamp < MAX_BLOCK_DELAY_SECONDS;
+    const result = this.getStatus(key, isHealthy, {
+      blockTimestamp,
+      nowTimestamp,
+    });
+
+    if (isHealthy) return result;
+    throw new HealthCheckError('Provider check failed', result);
+  }
+}

src/provider/provider.module.ts

@@ -79,9 +79,9 @@ const getProviderFactory = (SourceProvider: typeof JsonRpcProvider) => {
           }, MAX_TIME_WITHOUT_NEW_BLOCKS_MS);
         };
 
-        startDieTimer(-1);
-
         if (eventName === 'block') {
+          startDieTimer(-1);
+
           super.on(eventName, function (this: any, ...args) {
             startDieTimer(args[0]);
             return listener?.apply(this, args);

yarn.lock

@@ -1028,6 +1028,13 @@
     jsonc-parser "3.0.0"
     pluralize "8.0.0"
 
+"@nestjs/terminus@^8.0.1":
+  version "8.0.1"
+  resolved "https://registry.yarnpkg.com/@nestjs/terminus/-/terminus-8.0.1.tgz#96403795cbb6f14621c7504ee534fba336066643"
+  integrity sha512-UUSKK3/kO2gMUdXGJaOsj89pYA6OJMoKQPDx+zoEKQhA3+TPp0Js4ndWrtms5g6ydf3BfQqGQgEBEBjiq+0LWQ==
+  dependencies:
+    check-disk-space "3.0.1"
+
 "@nestjs/testing@^8.0.11":
   version "8.0.11"
   resolved "https://registry.yarnpkg.com/@nestjs/testing/-/testing-8.0.11.tgz#d41fc20a149dd919c9dfacb668f1c9cbc131f57c"
@@ -1985,6 +1992,11 @@ chardet@^0.7.0:
   resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
   integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
 
+[email protected]:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/check-disk-space/-/check-disk-space-3.0.1.tgz#c3798f3e4ec56dcc4d4b98dac0b770e2634efad7"
+  integrity sha512-wRG5ZihEZUXAKVCio5YkLgrHsHDrD+GjRBjxJvoJvN9wrze7EdvYGnOYJSsCyZ4b40jBa3lDmWDmESY9QDQGcg==
+
 [email protected], chokidar@^3.4.2:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75"