feat: revert telemetry instrumentation to otlp (#13)

.github/workflows/docker_build_push.yml

@@ -54,14 +54,14 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
+          VITE_RECAPTCHA_SITE_KEY=${{ vars.RECAPTCHA_SITE_KEY }}
           VITE_SUBNET_REGISTRATOR_CONTRACT_ADDRESS=${{ vars.SUBNET_REGISTRATOR_CONTRACT_ADDRESS }}
           VITE_TOPOS_CORE_PROXY_CONTRACT_ADDRESS=${{ vars.TOPOS_CORE_PROXY_CONTRACT_ADDRESS }}
           VITE_TOPOS_SUBNET_ENDPOINT_WS=${{ vars.TOPOS_SUBNET_ENDPOINT_WS }}
           VITE_TOPOS_SUBNET_ENDPOINT_HTTP=${{ vars.TOPOS_SUBNET_ENDPOINT_HTTP }}
-          VITE_TRACING_SERVICE_NAME=${{ vars.TRACING_SERVICE_NAME_FRONTEND }}
-          VITE_TRACING_SERVICE_VERSION=${{ github.ref_name }}
-          VITE_ELASTIC_APM_ENDPOINT=${{ vars.ELASTIC_APM_ENDPOINT }}
-          VITE_RECAPTCHA_SITE_KEY=${{ vars.RECAPTCHA_SITE_KEY }}
+          VITE_OTEL_SERVICE_NAME=${{ vars.OTEL_SERVICE_NAME }}
+          VITE_OTEL_SERVICE_VERSION=${{ vars.OTEL_SERVICE_VERSION }}
+          VITE_OTEL_EXPORTER_OTLP_ENDPOINT=${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
         cache-from: |
           type=registry,ref=${{ env.REGISTRY }}/${{ github.repository }}:build-cache-${{ env.GITHUB_REF_SLUG_URL }}-${{ github.workflow }}
           type=registry,ref=${{ env.REGISTRY }}/${{ github.repository }}:build-cache-main-${{ github.workflow }}

Dockerfile

@@ -1,13 +1,13 @@
 FROM node:lts-alpine
 
+ARG VITE_RECAPTCHA_SITE_KEY
 ARG VITE_SUBNET_REGISTRATOR_CONTRACT_ADDRESS
 ARG VITE_TOPOS_CORE_PROXY_CONTRACT_ADDRESS
 ARG VITE_TOPOS_SUBNET_ENDPOINT_WS
 ARG VITE_TOPOS_SUBNET_ENDPOINT_HTTP
-ARG VITE_TRACING_SERVICE_NAME
-ARG VITE_TRACING_SERVICE_VERSION
-ARG VITE_ELASTIC_APM_ENDPOINT
-ARG VITE_RECAPTCHA_SITE_KEY
+ARG VITE_OTEL_SERVICE_NAME
+ARG VITE_OTEL_SERVICE_VERSION
+ARG VITE_OTEL_EXPORTER_OTLP_ENDPOINT
 
 WORKDIR /usr/src/app
 

packages/backend/.env.example

@@ -1,6 +1,7 @@
 PORT=
-TRACING_SERVICE_NAME=
-TRACING_SERVICE_VERSION=
-ELASTIC_APM_ENDPOINT=
-ELASTIC_APM_TOKEN=
 THROTTLER_TTL_SECONDS=
+
+# telemetry
+OTEL_EXPORTER_OTLP_ENDPOINT=
+OTEL_SERVICE_NAME=
+OTEL_SERVICE_VERSION=

packages/backend/package.json

@@ -25,6 +25,11 @@
     "@nestjs/platform-express": "9.3.12",
     "@nestjs/serve-static": "^3.0.1",
     "@nestjs/throttler": "^4.2.1",
+    "@opentelemetry/api": "^1.6.0",
+    "@opentelemetry/exporter-metrics-otlp-proto": "^0.44.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.44.0",
+    "@opentelemetry/sdk-metrics": "^1.17.1",
+    "@opentelemetry/sdk-node": "^0.44.0",
     "class-transformer": "0.5.1",
     "class-validator": "0.14.0",
     "elastic-apm-node": "^3.49.1",

packages/backend/src/app/app.module.ts

@@ -5,10 +5,12 @@ import { ServeStaticModule } from '@nestjs/serve-static'
 import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler'
 import { join } from 'path'
 
+import { TelemetryModule } from '../telemetry/telemetry.module'
 import { FaucetModule } from '../faucet/faucet.module'
 
 @Module({
   imports: [
+    TelemetryModule,
     ConfigModule.forRoot({ isGlobal: true }),
     ServeStaticModule.forRoot({
       rootPath: join(__dirname, '..', '..', '..', 'frontend', 'dist'),

packages/backend/src/faucet/faucet.controller.ts

@@ -1,17 +1,38 @@
 import { Body, Controller, Headers, Post } from '@nestjs/common'
+import { context, metrics, propagation, trace } from '@opentelemetry/api'
 
 import { GetSubnetAssetsDto } from './faucet.dto'
 import { FaucetService } from './faucet.service'
 
 @Controller('faucet')
 export class FaucetController {
+  private _tracer = trace.getTracer('FaucetController')
+  private _meter = metrics.getMeter('FaucetController')
+  private _counter = this._meter.createCounter('get_subnet_assets.counter')
+
   constructor(private faucetService: FaucetService) {}
 
   @Post('getSubnetAssets')
   async getSubnetAssets(
     @Body() getSubnetAssetsDto: GetSubnetAssetsDto,
-    @Headers('traceparent') traceparent: string
+    @Headers('traceparent') traceparent?: string,
+    @Headers('tracestate') tracestate?: string
   ) {
-    return this.faucetService.getSubnetAssets(getSubnetAssetsDto, traceparent)
+    this._counter.add(1)
+
+    const activeContext = propagation.extract(context.active(), {
+      traceparent,
+      tracestate,
+    })
+
+    return context.with(activeContext, async () => {
+      return this._tracer.startActiveSpan('getSubnetAssets', async (span) => {
+        const receipts = await this.faucetService.getSubnetAssets(
+          getSubnetAssetsDto
+        )
+        span.end()
+        return receipts
+      })
+    })
   }
 }

packages/backend/src/faucet/faucet.module.ts

@@ -1,12 +1,10 @@
 import { Module } from '@nestjs/common'
 
-import { ApmModule } from '../apm/apm.module'
 import { FaucetController } from './faucet.controller'
 import { FaucetService } from './faucet.service'
 
 @Module({
   controllers: [FaucetController],
-  imports: [ApmModule],
   providers: [FaucetService],
 })
 export class FaucetModule {}

packages/backend/src/faucet/faucet.service.ts

@@ -1,60 +1,67 @@
 import { Injectable } from '@nestjs/common'
 import { ConfigService } from '@nestjs/config'
-import { providers, utils, Wallet } from 'ethers'
+import { SpanStatusCode, trace } from '@opentelemetry/api'
+import { ethers, providers, utils } from 'ethers'
 
 import { GetSubnetAssetsDto } from './faucet.dto'
 import { PROVIDER_ERRORS, WALLET_ERRORS } from './faucet.errors'
-import { ApmService } from 'src/apm/apm.service'
+import { getErrorMessage } from 'src/utils'
 
 @Injectable()
 export class FaucetService {
-  constructor(
-    private configService: ConfigService,
-    private apmService: ApmService
-  ) {}
-
-  async getSubnetAssets(
-    { address, subnetEndpoints }: GetSubnetAssetsDto,
-    traceparent?: string
-  ) {
-    const apmTransaction = this.apmService.startTransaction(
-      'faucetService.getSubnetAssets',
-      traceparent
-    )
-
-    apmTransaction.addLabels({
-      address,
-      subnetEndpoints: JSON.stringify(subnetEndpoints),
-    })
-
-    const promises = []
+  private _tracer = trace.getTracer('FaucetService')
+  constructor(private configService: ConfigService) {}
+
+  async getSubnetAssets({ address, subnetEndpoints }: GetSubnetAssetsDto) {
+    return this._tracer.startActiveSpan('getSubnetAssets', async (span) => {
+      span.setAttributes({
+        address,
+        subnetEndpoints,
+      })
+      const promises: Array<
+        Promise<Partial<ethers.providers.TransactionReceipt>>
+      > = []
+
+      for (const endpoint of subnetEndpoints) {
+        promises.push(
+          new Promise<Partial<ethers.providers.TransactionReceipt>>(
+            async (resolve, reject) => {
+              try {
+                const { logsBloom, ...receipt } = await this._sendTransaction(
+                  address,
+                  endpoint
+                )
+                span.setAttribute(
+                  `receipt-${endpoint.replace('.', '-')}`,
+                  JSON.stringify(receipt)
+                )
+                resolve(receipt)
+              } catch (error) {
+                const message = getErrorMessage(error)
+                span.setAttribute(
+                  `error-${endpoint.replace('.', '-')}`,
+                  message
+                )
+                reject(error)
+              }
+            }
+          )
+        )
+      }
 
-    for (const endpoint of subnetEndpoints) {
-      promises.push(
-        new Promise(async (resolve, reject) => {
-          try {
-            const { logsBloom, ...receipt } = await this._sendTransaction(
-              address,
-              endpoint
-            )
-            apmTransaction.addLabels({
-              [`receipt-${endpoint.replace('.', '-')}`]:
-                JSON.stringify(receipt),
-            })
-            resolve(receipt)
-          } catch (error) {
-            this.apmService.captureError(error)
-            reject(error)
-          }
+      return Promise.all(promises)
+        .then((receipts) => {
+          span.setStatus({ code: SpanStatusCode.OK })
+          return receipts
         })
-      )
-    }
-
-    const receipts = await Promise.all(promises)
-
-    apmTransaction.end()
-
-    return receipts
+        .catch((error) => {
+          const message = getErrorMessage(error)
+          span.setStatus({ code: SpanStatusCode.ERROR, message })
+        })
+        .finally(() => {
+          span.end()
+        })
+    })
   }
 
   private async _sendTransaction(address: string, subnetEndpoint: string) {
@@ -95,7 +102,6 @@ export class FaucetService {
         provider.on('debug', (data) => {
           if (data.error) {
             clearTimeout(timeoutId)
-            this.apmService.captureError(data.error)
             reject(new Error(PROVIDER_ERRORS.INVALID_ENDPOINT))
           }
         })
@@ -107,9 +113,11 @@ export class FaucetService {
     provider: providers.WebSocketProvider | providers.JsonRpcProvider
   ) {
     try {
-      return new Wallet(this.configService.get<string>('PRIVATE_KEY'), provider)
+      return new ethers.Wallet(
+        this.configService.get('PRIVATE_KEY') || '',
+        provider
+      )
     } catch (error) {
-      this.apmService.captureError(error)
       throw new Error(WALLET_ERRORS.INVALID_PRIVATE_KEY)
     }
   }

packages/backend/src/main.ts

@@ -1,9 +1,9 @@
 import { ValidationPipe } from '@nestjs/common'
 import { Logger } from '@nestjs/common'
 import { NestFactory } from '@nestjs/core'
+import { NestExpressApplication } from '@nestjs/platform-express'
 
 import { AppModule } from './app/app.module'
-import { NestExpressApplication } from '@nestjs/platform-express'
 
 async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule)

packages/backend/src/telemetry/telemetry.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common'
+
+import { TelemetryService } from './telemetry.service'
+
+@Module({
+  exports: [TelemetryService],
+  providers: [TelemetryService],
+})
+export class TelemetryModule {}

packages/backend/src/telemetry/telemetry.service.ts

@@ -0,0 +1,49 @@
+import { Injectable } from '@nestjs/common'
+
+// Uncomment to get otlp logs
+// import {
+//   diag,
+//   DiagConsoleLogger,
+//   DiagLogLevel,
+//   metrics,
+// } from '@opentelemetry/api'
+import { NodeSDK } from '@opentelemetry/sdk-node'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+
+const OTEL_EXPORTER_OTLP_ENDPOINT =
+  process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318'
+
+@Injectable()
+export class TelemetryService {
+  constructor() {
+    // Uncomment to get otlp logs
+    // diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG)
+
+    const otelSDK = new NodeSDK({
+      traceExporter: new OTLPTraceExporter({
+        url: `${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
+      }),
+      metricReader: new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter({
+          url: `${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/metrics`,
+        }),
+      }),
+      instrumentations: [],
+    })
+
+    otelSDK.start()
+
+    // gracefully shut down the SDK on process exit
+    process.on('SIGTERM', () => {
+      otelSDK
+        .shutdown()
+        .then(
+          () => console.log('SDK shut down successfully'),
+          (err) => console.log('Error shutting down SDK', err)
+        )
+        .finally(() => process.exit(0))
+    })
+  }
+}

packages/backend/src/utils/index.ts

@@ -0,0 +1,3 @@
+export function getErrorMessage(error: unknown) {
+  return error instanceof Error ? error.message : String(error)
+}