LiteProtocolTester application and docker compose bundle setup. (#2706)

faster image build with copy from pre-built binary cluster-id to 0 Added README.md documentation
waku-org · May 21, 2024 · 58aa5e6 · 58aa5e6
1 parent 7464684
commit 58aa5e6
Show file tree

Hide file tree

Showing 25 changed files with 7,026 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -66,3 +66,9 @@ coverage_html_report/
 
 # Wildcard
 *.ignore.*
+
+# Ignore all possible node runner directories
+**/keystore/
+**/rln_tree/
+**/certs/
+
diff --git a/Makefile b/Makefile
@@ -214,6 +214,10 @@ chat2bridge: | build deps librln
 	echo -e $(BUILD_MSG) "build/$@" && \
 		$(ENV_SCRIPT) nim chat2bridge $(NIM_PARAMS) waku.nims
 
+liteprotocoltester: | build deps librln
+	echo -e $(BUILD_MSG) "build/$@" && \
+		$(ENV_SCRIPT) nim liteprotocoltester $(NIM_PARAMS) waku.nims
+
 
 ################
 ## Waku tools ##

diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile
@@ -0,0 +1,58 @@
+# BUILD NIM APP ----------------------------------------------------------------
+  FROM rust:1.77.1-alpine3.18  AS nim-build
+
+  ARG NIMFLAGS
+  ARG MAKE_TARGET=liteprotocoltester
+  ARG NIM_COMMIT
+  ARG LOG_LEVEL=TRACE
+
+  # Get build tools and required header files
+  RUN apk add --no-cache bash git build-base pcre-dev linux-headers curl jq
+
+  WORKDIR /app
+  COPY . .
+
+  # workaround for alpine issue: https://github.com/alpinelinux/docker-alpine/issues/383
+  RUN apk update && apk upgrade
+
+  # Ran separately from 'make' to avoid re-doing
+  RUN git submodule update --init --recursive
+
+  # Slowest build step for the sake of caching layers
+  RUN make -j$(nproc) deps QUICK_AND_DIRTY_COMPILER=1 ${NIM_COMMIT}
+
+  # Build the final node binary
+  RUN make -j$(nproc) ${NIM_COMMIT} $MAKE_TARGET LOG_LEVEL=${LOG_LEVEL} NIMFLAGS="${NIMFLAGS}"
+
+
+  # PRODUCTION IMAGE -------------------------------------------------------------
+
+  FROM alpine:3.18 as prod
+
+  ARG MAKE_TARGET=liteprotocoltester
+
+  LABEL maintainer="[email protected]"
+  LABEL source="https://github.com/waku-org/nwaku"
+  LABEL description="Lite Protocol Tester: Waku light-client"
+  LABEL commit="unknown"
+  LABEL version="unknown"
+
+  # DevP2P, LibP2P, and JSON RPC ports
+  EXPOSE 30303 60000 8545
+
+  # Referenced in the binary
+  RUN apk add --no-cache libgcc pcre-dev libpq-dev
+
+  # Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
+  RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
+
+  # Copy to separate location to accomodate different MAKE_TARGET values
+  COPY --from=nim-build /app/build/$MAKE_TARGET /usr/bin/
+
+  # Copy migration scripts for DB upgrades
+  COPY --from=nim-build /app/migrations/ /app/migrations/
+
+  ENTRYPOINT ["/usr/bin/liteprotocoltester"]
+
+  # By default just show help if called without arguments
+  CMD ["--help"]
diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
@@ -0,0 +1,35 @@
+  # TESTING IMAGE --------------------------------------------------------------
+
+    ## NOTICE: This is a short cut build file for ubuntu users who compiles nwaku in ubuntu distro.
+    ##         This is used for faster turnaround time for testing the compiled binary.
+    ##         Prerequisites: compiled liteprotocoltester binary in build/ directory
+
+  FROM ubuntu:noble as prod
+
+  LABEL maintainer="[email protected]"
+  LABEL source="https://github.com/waku-org/nwaku"
+  LABEL description="Lite Protocol Tester: Waku light-client"
+  LABEL commit="unknown"
+  LABEL version="unknown"
+
+  # DevP2P, LibP2P, and JSON RPC ports
+  EXPOSE 30303 60000 8545
+
+  # Referenced in the binary
+  RUN apt-get update && apt-get install -y --no-install-recommends \
+      libgcc1 \
+      libpcre3 \
+      libpq-dev \
+      wget \
+      iproute2 \
+      && rm -rf /var/lib/apt/lists/*
+
+  # Fix for 'Error loading shared library libpcre.so.3: No such file or directory'
+  RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
+
+  COPY build/liteprotocoltester /usr/bin/
+
+  ENTRYPOINT ["/usr/bin/liteprotocoltester"]
+
+  # # By default just show help if called without arguments
+  CMD ["--help"]
diff --git a/apps/liteprotocoltester/README.md b/apps/liteprotocoltester/README.md
@@ -0,0 +1,87 @@
+# Waku - Lite Protocol Tester
+
+## Aim
+
+Testing reliability of light client protocols in different scale.
+Measure message delivery reliability and latency between light push client(s) and a filter client(s) node(s).
+
+## Concept of testing
+
+A tester node is configured either 'publisher' or 'receiver' and connects to a certain service node.
+All service protocols are disabled except for lightpush client or filter client. This way we would like to simulate
+a light client application.
+Each publisher pumps messages to the network in a preconfigured way (number of messages, frequency) while on the receiver side
+we would like to track and measure message losses, mis-ordered receives, late arrived messages and latencies.
+Ideally the tester nodes will connect to different edge of the network where we can gather more result from mulitple publishers
+and multiple receivers.
+
+Publishers are fill all message payloads with information about the test message and sender, helping the receiver side to calculate results.
+
+## Phases of development
+
+### Phase 1
+
+At the first phase we aims to demonstrate the concept of the testing all boundled into a docker-compose environment where we run
+one service (full)node and a publisher and a receiver node.
+At this stage we can only configure number of messages and fixed frequency of the message pump. We do not expect message losses and any significant latency hence the test setup is very simple.
+
+### Further plans
+
+- Add more configurability (randomized message sizes, usage of more content topics and support for static sharding).
+- Extend collected metrics and polish reporting.
+  - Add test metrics to graphana dashboard.
+- Support for static sharding and auto sharding for being able to test under different conditions.
+- ...
+
+## Usage
+
+### Phase 1
+
+Lite Protocol Tester application is built under name `liteprotocoltester` in apps/liteprotocoltester folder.
+
+Starting from nwaku repository root:
+```bash
+make liteprotocoltester
+cd apps/liteprotocoltester
+docker compose build
+docker compose up -d
+docker compose logs -f receivernode
+```
+
+## Configure
+
+### Environment variables for docker compose runs
+
+|   Variable     | Description | Default |
+| ---: | :--- | :--- |
+| NUM_MESSAGES   | Number of message to publish | 120 |
+| DELAY_MESSAGES | Frequency of messages in milliseconds | 1000 |
+| PUBSUB | Used pubsub_topic for testing | /waku/2/default-waku/proto |
+| CONTENT_TOPIC  | content_topic for testing | /tester/1/light-pubsub-example/proto |
+
+### Lite Protocol Tester application cli options
+
+|  Option     | Description | Default |
+| :--- | :--- | :--- |
+| --test_func | separation of PUBLISHER or RECEIVER mode | RECEIVER |
+| --service-node| Address of the service node to use for lightpush and/or filter service | - |
+| --num-messages | Number of message to publish | 120 |
+| --delay-messages | Frequency of messages in milliseconds | 1000 |
+| --pubsub-topic | Used pubsub_topic for testing | /waku/2/default-waku/proto |
+| --content_topic | content_topic for testing | /tester/1/light-pubsub-example/proto |
+| --cluster-id | Cluster id for the test | 0 |
+| --config-file | TOML configuration file to fine tune the light waku node<br>Note that some configurations (full node services) are not taken into account | - |
+| --nat |Same as wakunode "nat" configuration, appear here to ease test setup | any |
+| --rest-address | For convenience rest configuration can be done here | 127.0.0.1 |
+| --rest-port | For convenience rest configuration can be done here | 8654 |
+| --rest-allow-origin | For convenience rest configuration can be done here | * |
+| --log-level | Log level for the application | DEBUG |
+| --log-format | Logging output format (TEXT or JSON) | TEXT |
+
+
+
+### Docker image notice
+
+Please note that currently to ease testing and development tester application docker image is based on ubuntu and uses the externally pre-built binary of 'liteprotocoltester'.
+This speeds up image creation. Another dokcer build file is provided for proper build of boundle image.
+
diff --git a/apps/liteprotocoltester/docker-compose.yml b/apps/liteprotocoltester/docker-compose.yml
@@ -0,0 +1,160 @@
+version: "3.7"
+x-logging: &logging
+  logging:
+    driver: json-file
+    options:
+      max-size: 1000m
+
+# Environment variable definitions
+x-eth-client-address: &eth_client_address ${ETH_CLIENT_ADDRESS:-} # Add your ETH_CLIENT_ADDRESS after the "-"
+
+x-rln-environment: &rln_env
+  RLN_RELAY_CONTRACT_ADDRESS: ${RLN_RELAY_CONTRACT_ADDRESS:-0xF471d71E9b1455bBF4b85d475afb9BB0954A29c4}
+  RLN_RELAY_CRED_PATH: ${RLN_RELAY_CRED_PATH:-} # Optional: Add your RLN_RELAY_CRED_PATH after the "-"
+  RLN_RELAY_CRED_PASSWORD: ${RLN_RELAY_CRED_PASSWORD:-} # Optional: Add your RLN_RELAY_CRED_PASSWORD after the "-"
+
+x-test-running-conditions: &test_running_conditions
+  NUM_MESSAGES: ${NUM_MESSAGES:-120}
+  DELAY_MESSAGES: "${DELAY_MESSAGES:-1000}"
+  PUBSUB: ${PUBSUB:-}
+  CONTENT_TOPIC: ${CONTENT_TOPIC:-}
+
+# Services definitions
+services:
+  servicenode:
+    image: ${NWAKU_IMAGE:-harbor.status.im/wakuorg/nwaku:latest}
+    ports:
+      - 30304:30304/tcp
+      - 30304:30304/udp
+      - 9005:9005/udp
+      - 127.0.0.1:8003:8003
+      - 80:80 #Let's Encrypt
+      - 8000:8000/tcp #WSS
+      - 127.0.0.1:8645:8645
+    <<:
+      - *logging
+    environment:
+      DOMAIN: ${DOMAIN}
+      RLN_RELAY_CRED_PASSWORD: "${RLN_RELAY_CRED_PASSWORD}"
+      ETH_CLIENT_ADDRESS: *eth_client_address
+      EXTRA_ARGS: ${EXTRA_ARGS}
+      <<:
+        - *rln_env
+    volumes:
+      - ./run_service_node.sh:/opt/run_service_node.sh:Z
+      - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
+      - ./rln_tree:/etc/rln_tree/:Z
+      - ./keystore:/keystore:Z
+    entrypoint: sh
+    command:
+      - /opt/run_service_node.sh
+
+  publishernode:
+    image: waku.liteprotocoltester:latest
+    build:
+      context: ../..
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+    ports:
+      # - 30304:30304/tcp
+      # - 30304:30304/udp
+      # - 9005:9005/udp
+      # - 127.0.0.1:8003:8003
+      # - 80:80 #Let's Encrypt
+      # - 8000:8000/tcp #WSS
+      - 127.0.0.1:8646:8646
+    <<:
+      - *logging
+    environment:
+      DOMAIN: ${DOMAIN}
+      RLN_RELAY_CRED_PASSWORD: "${RLN_RELAY_CRED_PASSWORD}"
+      ETH_CLIENT_ADDRESS: *eth_client_address
+      EXTRA_ARGS: ${EXTRA_ARGS}
+      <<:
+        - *rln_env
+        - *test_running_conditions
+    volumes:
+      - ./run_tester_node.sh:/opt/run_tester_node.sh:Z
+      - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
+      - ./rln_tree:/etc/rln_tree/:Z
+      - ./keystore:/keystore:Z
+    entrypoint: sh
+    command:
+      - /opt/run_tester_node.sh
+      - SENDER
+    depends_on:
+      - servicenode
+    configs:
+      - source: cfg_tester_node.toml
+        target: config.toml
+
+  receivernode:
+    image: waku.liteprotocoltester:latest
+    build:
+      context: ../..
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+    ports:
+      # - 30304:30304/tcp
+      # - 30304:30304/udp
+      # - 9005:9005/udp
+      # - 127.0.0.1:8003:8003
+      # - 80:80 #Let's Encrypt
+      # - 8000:8000/tcp #WSS
+      - 127.0.0.1:8647:8647
+    <<:
+      - *logging
+    environment:
+      DOMAIN: ${DOMAIN}
+      RLN_RELAY_CRED_PASSWORD: "${RLN_RELAY_CRED_PASSWORD}"
+      ETH_CLIENT_ADDRESS: *eth_client_address
+      EXTRA_ARGS: ${EXTRA_ARGS}
+      <<:
+        - *rln_env
+        - *test_running_conditions
+    volumes:
+      - ./run_tester_node.sh:/opt/run_tester_node.sh:Z
+      - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
+      - ./rln_tree:/etc/rln_tree/:Z
+      - ./keystore:/keystore:Z
+    entrypoint: sh
+    command:
+      - /opt/run_tester_node.sh
+      - RECEIVER
+    depends_on:
+      - servicenode
+      - publishernode
+    configs:
+      - source: cfg_tester_node.toml
+        target: config.toml
+
+  prometheus:
+    image: docker.io/prom/prometheus:latest
+    volumes:
+      - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    ports:
+      - 127.0.0.1:9090:9090
+    depends_on:
+      - servicenode
+
+  grafana:
+    image: docker.io/grafana/grafana:latest
+    env_file:
+      - ./monitoring/configuration/grafana-plugins.env
+    volumes:
+      - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z
+      - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z
+      - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z
+      - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z
+      - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z
+      - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z
+      - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z
+    ports:
+      - 0.0.0.0:3000:3000
+    depends_on:
+      - prometheus
+
+configs:
+  cfg_tester_node.toml:
+    content: |
+      max-connections = 100