Merge remote-tracking branch 'origin/main' into affan/test-fix

.github/workflows/indexer-build-and-push-dev-staging.yml

@@ -4,8 +4,8 @@ on: # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-      - 'release/[a-z]+/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
-      - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/indexer/v1.x
+      - 'release/indexer/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
+      - 'release/indexer/v[0-9]+.x'  # e.g. release/indexer/v1.x
     # TODO(DEC-837): Customize github build and push to ECR by service with paths
 
 jobs:

.github/workflows/indexer-build-and-push-mainnet.yml

@@ -4,8 +4,8 @@ on: # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-      - 'release/[a-z]+/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
-      - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/indexer/v1.x
+      - 'release/indexer/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
+      - 'release/indexer/v[0-9]+.x'  # e.g. release/indexer/v1.x
     # TODO(DEC-837): Customize github build and push to ECR by service with paths
 
 jobs:

.github/workflows/indexer-build-and-push-testnet.yml

@@ -4,8 +4,8 @@ on: # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-      - 'release/[a-z]+/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
-      - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/indexer/v1.x
+      - 'release/indexer/v[0-9]+.[0-9]+.x'  # e.g. release/indexer/v0.1.x
+      - 'release/indexer/v[0-9]+.x'  # e.g. release/indexer/v1.x
     # TODO(DEC-837): Customize github build and push to ECR by service with paths
 
 jobs:

.github/workflows/protocol-benchmark.yml

@@ -0,0 +1,46 @@
+name: Protocol Benchmark
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    paths:
+      - 'protocol/**'
+  push:
+    branches:
+      - main
+      - 'release/protocol/v[0-9]+.[0-9]+.x'  # e.g. release/protocol/v0.1.x
+      - 'release/protocol/v[0-9]+.x'  # e.g. release/protocol/v1.x
+    paths:
+      - 'protocol/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./protocol
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Setup Golang
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.22
+      - name: Run Benchmarks
+        run: make benchmark | tee ./benchmark_output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'go'
+          output-file-path: ./protocol/benchmark_output.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: true
+          alert-threshold: '150%'
+          save-data-file: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}

.github/workflows/protocol-build-and-push-snapshot.yml

@@ -4,8 +4,8 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-      - 'release/[a-z]+/v[0-9]+.[0-9]+.x'  # e.g. release/protocol/v0.1.x
-      - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/protocol/v1.x
+      - 'release/protocol/v[0-9]+.[0-9]+.x'  # e.g. release/protocol/v0.1.x
+      - 'release/protocol/v[0-9]+.x'  # e.g. release/protocol/v1.x
 
 jobs:
   build-and-push-snapshot-dev:

.github/workflows/protocol-build-and-push.yml

@@ -4,8 +4,8 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - main
-      - 'release/[a-z]+/v[0-9]+.[0-9]+.x'  # e.g. release/protocol/v0.1.x
-      - 'release/[a-z]+/v[0-9]+.x'  # e.g. release/protocol/v1.x
+      - 'release/protocol/v[0-9]+.[0-9]+.x'  # e.g. release/protocol/v0.1.x
+      - 'release/protocol/v[0-9]+.x'  # e.g. release/protocol/v1.x
 
 jobs:
   build-and-push-dev:

.github/workflows/protocol-container-tests.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Build images
         run: DOCKER_BUILDKIT=1 make test-container-build
       - name: Run container tests

.github/workflows/protocol-exchange-tests.yml

@@ -30,7 +30,7 @@ jobs:
         name: Setup Golang
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       -
         name: Display go version
         run: go version

.github/workflows/protocol-lint.yml

@@ -45,6 +45,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Run golangci-lint
         run: make lint

.github/workflows/protocol-release.yml

@@ -26,7 +26,7 @@ jobs:
       - name: install go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.21'
+          go-version: '1.22'
           check-latest: true
       - name: Create Directory
         run: mkdir ./build

.github/workflows/protocol-sim.yml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - run: make build
@@ -39,7 +39,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - name: Install runsim
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - uses: actions/[email protected]
@@ -109,7 +109,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - uses: actions/[email protected]
@@ -159,7 +159,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - uses: actions/[email protected]
@@ -209,7 +209,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: Display go version
         run: go version
       - uses: actions/[email protected]

.github/workflows/protocol-test.yml

@@ -30,7 +30,7 @@ jobs:
         name: Setup Golang
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       -
         name: Ensure `go.mod` is up to date
         run: go mod tidy && git diff --exit-code
@@ -52,7 +52,7 @@ jobs:
         name: Setup Golang
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       -
         name: Ensure `go.mod` is up to date
         run: go mod tidy && git diff --exit-code
@@ -69,7 +69,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - uses: actions/checkout@v3
       - name: Install goveralls
         run: go install github.com/mattn/goveralls@latest
@@ -104,7 +104,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.21
+          go-version: 1.22
       - name: start localnet
         run: |
           DOCKER_BUILDKIT=1 make localnet-startd

indexer/Dockerfile.auxo.remote

@@ -6,6 +6,9 @@ WORKDIR /usr/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy auxo and imported packages
 COPY ./packages/base/ ./packages/base/
 COPY ./packages/v4-protos/ ./packages/v4-protos/
@@ -19,7 +22,7 @@ COPY tsconfig.json ./
 RUN npm install -g pnpm@6
 
 # Install npm modules using pnpm
-RUN pnpm install --loglevel warn --frozen-lockfile
+RUN pnpm install --loglevel warn --frozen-lockfile --unsafe-perm
 RUN pnpm run build:prod:all
 
 FROM public.ecr.aws/lambda/nodejs:16

indexer/Dockerfile.bazooka.remote

@@ -4,7 +4,10 @@ FROM public.ecr.aws/lambda/nodejs:16 as builder
 WORKDIR /usr/app
 
 # Copy pnpm lock and workspace and package.json from base directory
-COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
+COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./patches ./
+
+# Copy patches
+COPY ./patches ./patches
 
 # Copy bazooka and imported packages
 COPY ./packages/base/ ./packages/base/
@@ -22,7 +25,7 @@ COPY tsconfig.json ./
 RUN npm install -g pnpm@6
 
 # Install npm modules using pnpm
-RUN pnpm install --loglevel warn --frozen-lockfile
+RUN pnpm install --loglevel warn --frozen-lockfile --unsafe-perm
 RUN pnpm run build:prod:all
 
 FROM public.ecr.aws/lambda/nodejs:16

indexer/Dockerfile.postgres-package.local

@@ -11,6 +11,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from postgres and imported packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/postgres/package.json ./packages/postgres/
@@ -37,7 +40,7 @@ RUN chown dydx -R /home/dydx/app
 USER dydx
 
 # Install npm modules using pnpm
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 WORKDIR /home/dydx/app/packages/postgres
 

indexer/Dockerfile.service.local

@@ -13,6 +13,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from all packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/kafka/package.json ./packages/kafka/
@@ -42,7 +45,7 @@ COPY ./scripts/container-run.sh /home/dydx/app/services/$service/scripts/
 RUN chown dydx -R /home/dydx/app
 
 USER dydx
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 WORKDIR /home/dydx/app/services/$service
 

indexer/Dockerfile.service.remote

@@ -13,6 +13,9 @@ WORKDIR /home/dydx/app
 # Copy pnpm lock and workspace and package.json from base directory
 COPY ./pnpm-lock.yaml ./pnpm-workspace.yaml ./package.json ./
 
+# Copy patches
+COPY ./patches ./patches
+
 # Copy package.json from all packages being run
 COPY ./packages/base/package.json ./packages/base/
 COPY ./packages/kafka/package.json ./packages/kafka/
@@ -42,7 +45,7 @@ COPY ./scripts/container-run.sh /home/dydx/app/services/$service/scripts/
 RUN chown dydx -R /home/dydx/app
 
 # Install npm modules using pnpm
-RUN pnpm i --loglevel warn --production --frozen-lockfile
+RUN pnpm i --loglevel warn --production --frozen-lockfile --unsafe-perm
 
 USER dydx
 

indexer/README.md

@@ -107,6 +107,8 @@ pnpm run test:all
 
 If you change any logic, you'll have to re-build the services and packages before running unit tests.
 
+### To run a single test file:
+`cd services/{service_name} && pnpm build && pnpm test -- {test_name}`
 
 # Running Dockerfile locally
 TODO(DEC-671): Add e2e tests

indexer/package.json

@@ -13,8 +13,15 @@
     "lint:all": "pnpm run --parallel lint",
     "lint:fix:all": "pnpm run --parallel lint:fix",
     "coverage:all": "pnpm recursive run --workspace-concurrency=1 coverage",
-    "test:all": "pnpm recursive run --workspace-concurrency=1 test"
+    "test:all": "pnpm recursive run --workspace-concurrency=1 test",
+    "postinstall": "patch-package"
   },
   "author": "",
-  "license": "AGPL-3.0"
+  "license": "AGPL-3.0",
+  "dependencies": {
+    "@milahu/patch-package": "6.4.14"
+  },
+  "devDependencies": {
+    "@types/ws": "8.5.10"
+  }
 }

indexer/packages/base/src/logger.ts

@@ -5,7 +5,6 @@
 import winston from 'winston';
 
 import config from './config';
-import { redact } from './sanitization';
 import { InfoObject } from './types';
 
 // Fix types. The methods available depend on the levels used. We're using syslog levels, so these
@@ -38,8 +37,7 @@ const logger: LoggerExport = winston.createLogger({
     winston.format((info) => {
       return {
         ...info,           // info contains some symbols that are lost when the object is cloned.
-        ...redact(info),
-        error: info.error, // cloning with redact() may break the error object
+        error: info.error,
       };
     })(),
     winston.format.json(),