feat(mojaloop/3299): cl-tb: tigerbeetle into central-ledger standalone mode (#956)

.circleci/config.yml

@@ -211,10 +211,10 @@ jobs:
             # docker-compose build
 
             ## Lets pull only the Services needed for the Integration Test
-            docker-compose pull mysql kafka init-kafka
+            docker-compose pull mysql tigerbeetle kafka init-kafka
 
             ## Lets startup only the Services needed for the Integration Test
-            docker-compose up -d mysql kafka init-kafka
+            docker-compose up -d mysql tigerbeetle kafka init-kafka
 
             ## Check straight away to see if any containers have exited
             docker-compose ps
@@ -353,10 +353,10 @@ jobs:
           name: Build Docker local image
           command: |
             echo "Building Docker image: local"
-            docker build -t $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:local .
+            docker build -t mojaloop/$CIRCLE_PROJECT_REPONAME:local .
       - run:
           name: Save docker image to workspace
-          command: docker save -o /tmp/docker-image-local.tar $DOCKER_ORG/$CIRCLE_PROJECT_REPONAME:local
+          command: docker save -o /tmp/docker-image-local.tar mojaloop/$CIRCLE_PROJECT_REPONAME:local
       - persist_to_workspace:
           root: /tmp
           paths:

.dockerignore

@@ -15,3 +15,13 @@ Dockerfile
 LICENSE
 README.md
 sonar-project.properties
+
+# TigerBeetle
+tb/cluster_*.tigerbeetle
+
+# Mojaloop Central Ledger Stress Test
+stresstesting/target
+
+# jMeter
+stresstesting/jMeterOut
+stresstesting/test-plan/InData.json

.gitignore

@@ -64,3 +64,13 @@ devspace*
 *IGNORE*
 *ignore*
 !.npmignore
+
+# TigerBeetle
+tb/cluster_*.tigerbeetle
+
+# Mojaloop Central Ledger Stress Test
+stresstesting/target
+
+# jMeter
+stresstesting/jMeterOut
+stresstesting/test-plan/InData.json

Dockerfile

@@ -2,7 +2,7 @@ FROM node:16.15.0-alpine as builder
 WORKDIR /opt/app
 
 RUN apk --no-cache add git
-RUN apk add --no-cache -t build-dependencies make gcc g++ python3 libtool libressl-dev openssl-dev autoconf automake \
+RUN apk add --no-cache -t build-dependencies make gcc g++ python3 libtool libressl-dev openssl-dev autoconf automake wget \
     && cd $(npm root -g)/npm \
     && npm config set unsafe-perm true \
     && npm install -g node-gyp
@@ -16,10 +16,34 @@ COPY config /opt/app/config
 COPY migrations /opt/app/migrations
 COPY seeds /opt/app/seeds
 COPY test /opt/app/test
+COPY tb /opt/app/tb
+
+# TigerBeetle
+## Init
+#RUN whoami
+
+#WORKDIR /opt/central-ledger/tb
+#USER root
+#RUN ./tb/tigerbeetle init --cluster=0 --replica=0 --directory=./tb
+#RUN ./tb/tigerbeetle init --cluster=0 --replica=1 --directory=./tb
+#RUN ./tb/tigerbeetle init --cluster=0 --replica=2 --directory=./tb
+RUN ls -ltra /opt/app/tb
+RUN ls -ltra /opt/app
+RUN cp /opt/app/test/run_test_integration.sh /opt/app/run_tests.sh
+RUN chmod 777 /opt/app/run_tests.sh
+
+#RUN chmod 777 *
+
+## Start
+#USER node
+
+#WORKDIR /opt/central-ledger
+
 
 FROM node:16.15.0-alpine
 WORKDIR /opt/app
 
+# CL Core
 # Create empty log file & link stdout to the application log file
 RUN mkdir ./logs && touch ./logs/combined.log
 RUN ln -sf /dev/stdout ./logs/combined.log
@@ -31,5 +55,17 @@ USER ml-user
 COPY --chown=ml-user --from=builder /opt/app .
 RUN npm prune --production
 
+# Start TB
+#RUN ./tb/tigerbeetle start --cluster=0 --replica=0 --directory=tb --addresses=5001,5002,5003 &> tb-r-1.log &
+#RUN ./tb/tigerbeetle start --cluster=0 --replica=1 --directory=tb --addresses=5001,5002,5003 &> tb-r-2.log &
+#RUN ./tb/tigerbeetle start --cluster=0 --replica=2 --directory=tb --addresses=5001,5002,5003 &> tb-r-3.log &
+
+# Node
 EXPOSE 3001
+
+# TigerBeetle
+EXPOSE 5001
+EXPOSE 5002
+EXPOSE 5003
+
 CMD ["npm", "run", "start"]

GOALS.md

@@ -0,0 +1,31 @@
+# Mojaloop Central Ledger using TigerBeetle
+TigerBeetle drop-in for Mojaloop `central-ledger`.
+
+See; https://github.com/mojaloop/central-ledger
+
+# Goals
+
+1. Drop in replacement for the Mojaloop `central-ledger` v13.x
+2. Compatible with Ref 2.0 Mojaloop architecture;
+    1. https://miro.com/app/board/o9J_lJyA1TA=/
+    2. https://docs.mojaloop.io/reference-architecture-doc/refarch/
+
+# Dependencies
+- ~~NodeJS >= 14.0.0 (cl is currently on `12.16.0-alpine`)~~
+- https://github.com/coilhq/tigerbeetle-node
+
+# Run
+```shell
+git clone [email protected]:coilhq/moja-ledger-beetle.git
+cd tiger-beetle/proto-beetle
+npm install
+dd < /dev/zero bs=1048576 count=256 > journal
+scripts/create-transfers
+node server
+
+# In another tab:
+cd tiger-beetle/proto-beetle
+time node stress
+```
+
+

README.md

@@ -76,21 +76,21 @@ Tests include unit, functional, and integration.
 Running the tests:
 
 ```bash
-    npm run test:all
+npm run test:all
 ```
 
 Tests include code coverage via istanbul. See the test/ folder for testing scripts.
 
 ### Running Integration Tests interactively
 
-If you want to run integration tests in a repetitive manner, you can startup the test containers using `docker-compose` via one of the following methods:
+If you want to run integration tests in a repetitive manner, you can start-up the test containers using `docker-compose` via one of the following methods:
 
 - Running locally
 
     Start containers required for Integration Tests
 
     ```bash
-    docker-compose -f docker-compose.yml up -d kafka mysql
+    docker-compose -f docker-compose.yml up -d kafka mysql tigerbeetle
     ```
 
     Run wait script which will report once all required containers are up and running
@@ -136,6 +136,11 @@ If you want to run integration tests in a repetitive manner, you can startup the
     npm run test:int
   ```
 
+- Removing the CL Image
+```shell
+docker rmi central-ledger_central-ledger
+```
+
 ### Running Functional Tests
 
 If you want to run functional tests locally utilizing the [ml-core-test-harness](https://github.com/mojaloop/ml-core-test-harness), you can run the following commands:

docker-compose.integration.yml

@@ -18,6 +18,7 @@ services:
       - ./docker/central-ledger/default.json:/opt/app/config/default.json
       - ./test:/opt/app/test
       - ./src:/opt/app/src
+      # - ./tb:/opt/central-ledger/tb
       # - ./node_modules:/opt/app/node_modules
     environment:
       - CLEDG_MONGODB__DISABLED=true

docker-compose.yml

@@ -95,6 +95,37 @@ services:
       start_period: 40s
       interval: 30s
 
+  tigerbeetle-setup:
+    image: ghcr.io/tigerbeetledb/tigerbeetle:debug
+    container_name: cl_tigerbeetle_setup
+    command: "format --cluster=0 --replica=0 --replica-count=1 /data/0_0.tigerbeetle"
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - ./docker/tigerbeetle:/data
+
+  tigerbeetle:
+    image: ghcr.io/tigerbeetledb/tigerbeetle:debug
+    container_name: cl_tigerbeetle
+    command: "start --addresses=0.0.0.0:5001 /data/0_0.tigerbeetle"
+    restart: unless-stopped
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - ./docker/tigerbeetle:/data
+    networks:
+      - cl-mojaloop-net
+    ports:
+      - 5001:5001
+    depends_on:
+      - tigerbeetle-setup
+    healthcheck:
+      test: ["CMD", "/tigerbeetle" ,"version"]
+      timeout: 20s
+      retries: 10
+      start_period: 40s
+      interval: 30s
+
   mockserver:
     image: jamesdbloom/mockserver
     container_name: mockserver

docker/central-ledger/default.json

@@ -77,6 +77,16 @@
   },
   "INTERNAL_TRANSFER_VALIDITY_SECONDS": "432000",
   "ENABLE_ON_US_TRANSFERS": false,
+  "TIGERBEETLE" : {
+    "ENABLED" : true,
+    "ENABLE_BATCHING" : false,
+    "DISABLE_SQL" : false,
+    "BATCH_MAX_SIZE" : 2048,
+    "CLUSTER" : 0,
+    "REPLICA_ENDPOINT_01" : "tigerbeetle:5001",
+    "REPLICA_ENDPOINT_02" : "",
+    "REPLICA_ENDPOINT_03" : ""
+  },
   "CACHE": {
     "CACHE_ENABLED": false,
     "MAX_BYTE_SIZE": 10000000,

docker/tigerbeetle/Dockerfile

@@ -0,0 +1,36 @@
+# Since we support arm64, take advantage of Docker and Zig's cross
+# compilation, rather than running under qemu.
+# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
+FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS build
+
+RUN  apt-get update \
+  && apt-get install -y wget xz-utils
+
+WORKDIR /opt/beta-beetle
+
+# Copy install_zig by itself, to try and keep cache hits as high as possible.
+RUN mkdir scripts
+COPY scripts/install_zig.sh ./scripts
+
+ENV PATH="${PATH}:/opt/beta-beetle/zig"
+
+# Allows passing in --build-arg DEBUG=true during builds (needed by Github CI for generating debug image)
+ARG DEBUG=false
+
+# With our multiplatform builds, Docker will only repeat steps after the ARG TARGETPLATFORM.
+# So, explicitly install zig before then, to not need to do it twice.
+RUN ./scripts/install_zig.sh
+
+COPY scripts ./scripts
+COPY src ./src
+COPY build.zig ./build.zig
+
+ARG TARGETPLATFORM
+RUN test ${TARGETPLATFORM} = "linux/arm64" && CPU="baseline+aes+neon" || CPU="x86_64_v3+aes"; CI=true ./scripts/install.sh
+
+FROM alpine:3.17 AS release
+WORKDIR /
+
+COPY --from=build /opt/beta-beetle/tigerbeetle ./tigerbeetle
+
+ENTRYPOINT ["/tigerbeetle"]

migrations/960100_tigerBeetleIlpPacket.js

@@ -0,0 +1,46 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Coil
+ - Jason Bruwer <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = async (knex) => {
+  return await knex.schema.hasTable('tigerBeetleIlpPacket').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('tigerBeetleIlpPacket', (t) => {
+        t.string('transferId', 36).primary().notNullable()
+        t.text('value').notNullable()
+        t.string('ilpCondition', 256).notNullable()
+        t.string('ilpFulfilment', 256).nullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+        t.dateTime('createdDateIlpFulfilment').nullable()
+        t.index('transferId')
+      })
+    }
+  })
+}
+
+exports.down = function (knex) {
+  return knex.schema.dropTableIfExists('tigerBeetleIlpPacket')
+}

migrations/960101_tigerBeetleTransferExtension.js

@@ -0,0 +1,47 @@
+/*****
+ License
+ --------------
+ Copyright © 2017 Bill & Melinda Gates Foundation
+ The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ Contributors
+ --------------
+ This is the official list of the Mojaloop project contributors for this file.
+ Names of the original copyright holders (individuals or organizations)
+ should be listed with a '*' in the first column. People who have
+ contributed from an organization can be listed under the organization
+ that actually holds the copyright for their contributions (see the
+ Gates Foundation organization for an example). Those individuals should have
+ their names indented and be marked with a '-'. Email address can be added
+ optionally within square brackets <email>.
+ * Gates Foundation
+ - Name Surname <[email protected]>
+
+ * Coil
+ *  - Jason Bruwer <[email protected]>
+ --------------
+ ******/
+
+'use strict'
+
+exports.up = async (knex) => {
+  return await knex.schema.hasTable('tigerBeetleTransferExtension').then(function(exists) {
+    if (!exists) {
+      return knex.schema.createTable('tigerBeetleTransferExtension', (t) => {
+        t.bigIncrements('transferExtensionId').primary().notNullable()
+        t.string('transferId', 36).notNullable()
+        t.string('transferFulfilmentId', 36).defaultTo(null).nullable()
+        t.string('key', 128).notNullable()
+        t.text('value').notNullable()
+        t.dateTime('createdDate').defaultTo(knex.fn.now()).notNullable()
+        t.index('transferId')
+        t.index('transferFulfilmentId')
+      })
+    }
+  })
+}
+
+exports.down = function (knex) {
+  return knex.schema.dropTableIfExists('tigerBeetleTransferExtension')
+}