Add in Monitoring

README.md

@@ -136,6 +136,20 @@ pnpm env:development build --filter=chain
 pnpm env:development sequencer:start --filter=chain
 ```
 
+### Observability
+
+Protokit has the ability to report metrics, logs and traces to a Grafana instance for visualisation.
+These can be configured by the following environment variables
+```zsh
+OPEN_TELEMETRY_TRACING_URL=
+OPEN_TELEMETRY_TRACING_ENABLED=
+
+OPEN_TELEMETRY_METRICS_URL=
+OPEN_TELEMETRY_METRICS_ENABLED=
+OPEN_TELEMETRY_METRICS_SCRAPING_FREQUENCY=
+````
+Note that the functionality is not configured for the `in-memory` mode.
+
 ### Running the UI
 
 ```zsh
@@ -230,11 +244,43 @@ The caddy reverse-proxy automatically uses https for all connections, use this g
 
 <https://caddyserver.com/docs/running#local-https-with-docker>
 
+### Monitoring
+
+Protokit offers monitoring via three different kinds of data and a collection of preconfigured services: 
+- Logs via Promtail and Loki
+- Metrics via OpenTelemetry and Prometheus
+- Traces via OpenTelemetry, OTel Collector and Tempo
+- Dashboard via Grafana
+
+#### Development
+
+In Development mode, monitoring is disabled by default.
+
+To enabled, edit the `development/.env` file in the following way:
+1. Add the monitoring profile to `COMPOSE_PROFILES`
+2. Uncomment `...metricsSequencerModules` in the sequencer's module definition. 
+Important: This has to be in front of all other modules (i.e. has to be first in the modules record)
+3. Uncomment `...metricsSequencerModulesConfig` in the configuration call.
+
+Then, run `pnpm env:development docker:up` like usual. This should start all the services needed for monitoring.
+Grafana is available at `localhost:3000`.
+
+Note: Logs are currently not available without docker, since promtail is only configured to pick up container logs
+
+#### Sovereign
+
+In Sovereign mode, monitoring is configured by default.
+
+Grafana is reachable under `localhost/grafana`.
+
+If you want to remove the monitoring services, remove the docker profile `monitoring` from the `.env` file and remove the `OpenTelemetryServer` configuration
+
+More information about monitoring can be found [here](https://github.com/proto-kit/framework/pull/272).
 
 ## Building the framework from source
 
 1. Make sure the framework is located under ../framework from the starter-kit's location
-2. Adapt your starter-kit's package.json to use the file:// references to framework
+2. Adapt your starter-kit's `packages/chain` and `apps/web` package.json to use the file:// references to framework, including
+references to `o1js` and `tsyringe`. Important: Make sure to update references in both chain and web, otherwise the location of the node_modules will be different and lead to errors
 3. Go into the framework folder, and build a docker image containing the sources with `docker build -f ./packages/deployment/docker/development-base/Dockerfile -t protokit-base .`
-
-4. Comment out the first line of docker/base/Dockerfile to use protokit-base
+4. Replace the first line of `docker/base/Dockerfile` and `docker/web/Dockerfile` to use `FROM protokit-base:latest`

docker/base/Dockerfile

@@ -1,5 +1,5 @@
 # For regular deployment
-FROM node:18
+FROM protokit-base
 
 # For locally built and imported framework references
 # Make sure you build the base-image first
@@ -17,7 +17,7 @@ COPY ./packages/chain/package.json ./packages/chain/package.json
 COPY ./packages/eslint-config-custom/package.json ./packages/eslint-config-custom/package.json
 COPY ./apps/web/package.json ./apps/web/package.json
 
-RUN corepack enable
+RUN npm i -g [email protected]
 
 COPY ./tsconfig.json .
 COPY ./packages/chain/tsconfig.json ./packages/chain/tsconfig.json

docker/base/entry.sh

@@ -25,4 +25,4 @@ cd /app
 
 echo $@
 # We can't use start:headless here bcs that task hardcodes the .js file, we want it to be extensible
-node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads --es-module-specifier-resolution=node $@
+node --experimental-vm-modules --experimental-wasm-modules --es-module-specifier-resolution=node $@

docker/docker-compose.yml

@@ -7,3 +7,4 @@ include:
   - ./indexer/docker-compose.yml
   - ./processor-persistence/docker-compose.yml
   - ./processor/docker-compose.yml
+  - ./monitoring/docker-compose.yml

docker/indexer/docker-compose.yml

@@ -43,6 +43,9 @@ services:
       [
         "./packages/chain/dist/start.js start ./environments/${PROTOKIT_ENV_FOLDER}/indexer.config.js",
       ]
+    labels:
+      logging: "promtail"
+      logging_jobname: "indexer"
 
 networks:
   indexer-net:

docker/monitoring/docker-compose.yml

@@ -0,0 +1,102 @@
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    profiles:
+      - monitoring
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-remote-write-receiver'
+      - '--enable-feature=native-histograms'
+    networks:
+      - monitoring
+      - chain-net
+    ports:
+      - "9090:9090"
+    restart: unless-stopped
+    volumes:
+      - ./prometheus/${PROTOKIT_ENV_FOLDER}:/etc/prometheus
+      - ../data/prometheus:/prometheus
+
+  loki:
+    image: grafana/loki:latest
+    container_name: loki
+    profiles:
+      - monitoring
+    networks:
+      - monitoring
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: promtail
+    profiles:
+      - monitoring
+    networks:
+      - monitoring
+    volumes:
+      - ./promtail/promtail.yaml:/etc/promtail/docker-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/docker-config.yaml
+    depends_on:
+      - loki
+
+  otel-collector:
+    image: otel/opentelemetry-collector:0.86.0
+    container_name: otel-collector
+    profiles:
+      - monitoring
+    networks:
+      - monitoring
+      - chain-net
+    command: [ "--config=/etc/otel-collector.yaml" ]
+    volumes:
+      - ./otel-collector/otel-collector.yaml:/etc/otel-collector.yaml
+    ports:
+      - "4318:4317"
+
+  tempo:
+    image: grafana/tempo:latest
+    container_name: tempo
+    networks:
+      - monitoring
+    profiles:
+      - monitoring
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo/tempo.yaml:/etc/tempo.yaml
+      - ../data/tempo:/var/tempo
+    ports:
+      - "3200:3200" # tempo
+      - "4317:4317" # otlp grpc
+
+  grafana:
+    image: grafana/grafana:11.0.1
+    container_name: grafana
+    profiles:
+      - monitoring
+    restart: unless-stopped
+    networks:
+      - monitoring
+    ports:
+      - "3001:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER}
+      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
+      - GF_SECURITY_ALLOW_EMBEDDING=true
+    volumes:
+      - ./grafana:/etc/grafana/provisioning/datasources
+      - ../data/grafana:/var/lib/grafana
+    depends_on:
+      - loki
+      - tempo
+      - prometheus
+    labels:
+      logging: "promtail"
+      logging_jobname: "grafana"
+
+networks:
+  monitoring:

docker/monitoring/grafana/grafana-datasources.yaml

@@ -0,0 +1,33 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    url: http://prometheus:9090
+    isDefault: true
+    access: proxy
+    editable: true
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1
+    url: http://loki:3100
+    basicAuth: false
+    version: 1
+    editable: false
+  - name: Tempo
+    type: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo:3200
+    basicAuth: false
+    version: 1
+    editable: false
+    apiVersion: 1
+    uid: tempo
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      streamingEnabled:
+        search: true

docker/monitoring/otel-collector/otel-collector.yaml

@@ -0,0 +1,16 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+exporters:
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+  debug:
+    verbosity: detailed
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]

docker/monitoring/prometheus/development/prometheus.yml

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: []
+      scheme: http
+      timeout: 10s
+      api_version: v2
+scrape_configs:
+  - job_name: prometheus
+    honor_timestamps: true
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets:
+          - localhost:9090
+  - job_name: sequencer
+    honor_timestamps: true
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets:
+          - host.docker.internal:4320

docker/monitoring/prometheus/sovereign/prometheus.yml

@@ -0,0 +1,30 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: []
+      scheme: http
+      timeout: 10s
+      api_version: v2
+scrape_configs:
+  - job_name: prometheus
+    honor_timestamps: true
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets:
+          - localhost:9090
+  - job_name: sequencer
+    honor_timestamps: true
+    scrape_interval: 15s
+    scrape_timeout: 10s
+    metrics_path: /metrics
+    scheme: http
+    static_configs:
+      - targets:
+          - starterkit-sequencer:4320

docker/monitoring/promtail/promtail.yaml

@@ -0,0 +1,26 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: flog_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        filters:
+          - name: label
+            values: ["logging=promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'