Adds graphql schema command + OpenTelemetry integration

.env

@@ -19,3 +19,10 @@ RPC_ENDPOINT=wss://rpc.joystream.org:9944
 # Uncommenting this line enables the debug mode
 # More info at https://docs.subsquid.io/basics/logging/
 #SQD_DEBUG=*
+
+# =====================================================================================
+
+## Telemetry
+# yes/no
+TELEMETRY_ENABLED=no
+TELEMETRY_ENDPOINT=http://apm-server:8200

.github/workflows/docker.yml

@@ -38,16 +38,9 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      # docker/build-push-action doc:
-      # Be careful because any file mutation in the steps that precede the
-      # build step will be ignored, including processing of the .dockerignore file
-      # since the context is based on the Git reference. However, you can use
-      # the Path context using the context input alongside the actions/checkout action
-      # to remove this restriction.
       - name: Build storage-squid
         uses: docker/build-push-action@v3
         with:
-          # Do not use local dir context to ensure we can build from a commit directly
           context: .
           file: Dockerfile
           push: false
@@ -58,3 +51,8 @@ jobs:
         run: |
           docker image tag joystream/storage-squid:latest joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }}
           docker push joystream/storage-squid:${{ steps.extract_version.outputs.squid_version }}
+
+      - name: Push storage-squid latest tag
+        if: github.ref == 'refs/heads/master'
+        run: |
+          docker push joystream/storage-squid:latest

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 1.1.0
+
+- remove unused dependencies from `package.json` file.
+- add opentelemetry integration.
+- adds `get-graphql-schema` npm command.
+- **FIX**: convert `ipfsHash` to string.
+
 # 1.0.0
 
 - Initial release of `storage-squid` package.

Dockerfile

@@ -39,6 +39,7 @@ ADD db db
 ADD assets assets
 ADD schema schema
 ADD scripts scripts
+ADD opentelemetry opentelemetry
 ENV PROCESSOR_PROMETHEUS_PORT 3000
 EXPOSE 3000
 EXPOSE 4000

Makefile

@@ -11,9 +11,6 @@ build:
 build-docker:
 	@docker build . -t joystream/storage-squid
 
-serve:
-	@npx squid-graphql-server --subscriptions
-
 migrate:
 	@npx squid-typeorm-migration apply
 

docker-compose.yml

@@ -50,15 +50,15 @@ services:
       - .env
       - .docker.env
     environment:
-      - SQD_TRACE=authentication
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
     depends_on:
       - squid_db
     volumes:
       - type: bind
         source: .
         target: /storage-squid
     working_dir: /storage-squid
-    command: ['make', 'serve']
+    command: ['npm', 'run', 'graphql-server-start']
     ports:
       - '${GQL_PORT}:${GQL_PORT}'
     networks:

opentelemetry/.env

@@ -0,0 +1,8 @@
+# Required env variables for the Elasticsearch exporters
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+OTEL_RESOURCE_ATTRIBUTES="service.name=squid-graphql-server,deployment.environment=production"
+OTEL_METRICS_EXPORTER="otlp"
+
+# Optional env vars to configure the opentelemetry exporters
+OTEL_MAX_QUEUE_SIZE=8192        # 4 times of default queue size
+OTEL_MAX_EXPORT_BATCH_SIZE=1024 # 2 times of default batch size

opentelemetry/index.js

@@ -0,0 +1,58 @@
+const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
+
+/**
+ * Error: "@opentelemetry/instrumentation-grpc Module @grpc/grpc-js has
+ * been loaded before @opentelemetry/instrumentation-grpc so it might not work,
+ * please initialize it before requiring @grpc/grpc-js"
+ *
+ * Fix: "call getNodeAutoInstrumentations() before require('@opentelemetry/sdk-node');"
+ */
+//
+
+// Disable DNS instrumentation, because the instrumentation does not correctly patches `dns.lookup` function
+// if the function is converted to a promise-based method using `utils.promisify(dns.lookup)`
+// See: https://github.com/Joystream/joystream/pull/4779#discussion_r1262515887
+const instrumentations = getNodeAutoInstrumentations({
+  '@opentelemetry/instrumentation-dns': { enabled: false },
+})
+
+const { NodeSDK } = require('@opentelemetry/sdk-node')
+const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto')
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-proto')
+const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-node')
+
+const path = require('path')
+require('dotenv').config({ path: path.resolve(__dirname, `.env`) })
+
+// For troubleshooting, set the log level to DiagLogLevel.DEBUG
+diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO)
+
+function addInstrumentation() {
+  const instrumentation = new NodeSDK({
+    spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter(), {
+      maxQueueSize: parseInt(process.env.OTEL_MAX_QUEUE_SIZE || '8192'),
+      maxExportBatchSize: parseInt(process.env.OTEL_MAX_EXPORT_BATCH_SIZE || '1024'),
+    }),
+    metricReader: new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter(),
+    }),
+    instrumentations,
+  })
+
+  // Start Opentelemetry NodeJS Instrumentation
+  diag.info('Starting tracing...')
+  instrumentation.start()
+
+  // gracefully shut down the SDK on process exit
+  process.on('SIGTERM', () => {
+    instrumentation
+      .shutdown()
+      .then(() => console.log('Tracing terminated'))
+      .catch((error) => console.log('Error terminating tracing', error))
+      .finally(() => process.exit(0))
+  })
+}
+
+addInstrumentation()