From 55c6b59385a316876624ec1a406764c9138bcb1e Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 16:53:42 +0700 Subject: [PATCH 1/9] wip: benchmark time with benchmark.js --- packages/object/benchmark/hashgraph.bench.ts | 70 ++++++++++++++++++++ packages/object/benchmark/output.txt | 0 packages/object/package.json | 1 + packages/object/tsconfig.json | 2 +- 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 packages/object/benchmark/hashgraph.bench.ts create mode 100644 packages/object/benchmark/output.txt diff --git a/packages/object/benchmark/hashgraph.bench.ts b/packages/object/benchmark/hashgraph.bench.ts new file mode 100644 index 00000000..6c902169 --- /dev/null +++ b/packages/object/benchmark/hashgraph.bench.ts @@ -0,0 +1,70 @@ +import Benchmark from "benchmark"; +import { AddWinsSet } from "../../blueprints/src/AddWinsSet/index.js"; +import { TopologyObject } from "../src/index.js"; + +function benchmarkForAddWinSet( + name: string, + numCROs: number, + verticesPerCRO: number, + mergeFn: boolean, +) { + return suite.add(name, () => { + const objects: TopologyObject[] = []; + for (let i = 0; i < numCROs; i++) { + const obj: TopologyObject = new TopologyObject( + `peer${i + 1}`, + new AddWinsSet(), + ); + const cro = obj.cro as AddWinsSet; + for (let j = 0; j < verticesPerCRO; j++) { + if (i % 3 === 2) { + cro.add(j); + cro.remove(j); + } else if (i % 3 === 1) { + cro.remove(j); + cro.add(j); + } else { + cro.add(j); + } + } + objects.push(obj); + } + + if (mergeFn) { + for (let i = 0; i < objects.length; i++) { + for (let j = 0; j < objects.length; j++) { + if (i !== j) { + objects[i].merge(objects[j].hashGraph.getAllVertices()); + } + } + } + } + }); +} + +const suite = new Benchmark.Suite(); + +benchmarkForAddWinSet("Create HashGraph with 1000 vertices", 1, 1000, false); + +benchmarkForAddWinSet( + "Create 2 CROs (1000 vertices each) and Merge", + 2, + 1000, + true, +); + +benchmarkForAddWinSet( + "Create 5 CROs (1000 vertices each) and Merge", + 5, + 1000, + true, +); + +suite + .on("cycle", (event) => { + console.log(String(event.target)); + }) + .on("complete", function () { + console.log(`Fastest is ${this.filter("fastest").map("name")}`); + }) + .run({ async: true }); diff --git a/packages/object/benchmark/output.txt b/packages/object/benchmark/output.txt new file mode 100644 index 00000000..e69de29b diff --git a/packages/object/package.json b/packages/object/package.json index ede49451..b1145dcf 100644 --- a/packages/object/package.json +++ b/packages/object/package.json @@ -11,6 +11,7 @@ "files": [ "src", "dist", + "benchmark", "!dist/test", "!**/*.tsbuildinfo" ], diff --git a/packages/object/tsconfig.json b/packages/object/tsconfig.json index 48e3023f..b53d3d8e 100644 --- a/packages/object/tsconfig.json +++ b/packages/object/tsconfig.json @@ -8,5 +8,5 @@ "path": "../logger" } ], - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "benchmarks/**/*.ts"] } From 97c325024c0b3e3becdeccb67079aad6729012df Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 18:05:11 +0700 Subject: [PATCH 2/9] feat: add benchmark script --- packages/object/benchmark/output.txt | 0 packages/object/package.json | 8 ++++++-- packages/object/tests/causallyrelated.bench.ts | 2 +- .../{benchmark => tests}/hashgraph.bench.ts | 18 +++++++++--------- packages/object/tsconfig.json | 2 +- 5 files changed, 17 insertions(+), 13 deletions(-) delete mode 100644 packages/object/benchmark/output.txt rename packages/object/{benchmark => tests}/hashgraph.bench.ts (86%) diff --git a/packages/object/benchmark/output.txt b/packages/object/benchmark/output.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/packages/object/package.json b/packages/object/package.json index b1145dcf..dbc69b4d 100644 --- a/packages/object/package.json +++ b/packages/object/package.json @@ -27,10 +27,14 @@ "build": "tsc -b", "clean": "rm -rf dist/ node_modules/", "prepack": "tsc -b", - "test": "vitest" + "test": "vitest", + "benchmark-hashgraph": "npx tsx tests/hashgraph.bench.ts" }, "devDependencies": { - "assemblyscript": "^0.27.29" + "assemblyscript": "^0.27.29", + "benchmark": "^2.1.4", + "ts-node": "^10.9.2", + "tsx": "4.19.1" }, "dependencies": { "@bufbuild/protobuf": "^2.0.0", diff --git a/packages/object/tests/causallyrelated.bench.ts b/packages/object/tests/causallyrelated.bench.ts index 8f9e1a73..1eaf0963 100644 --- a/packages/object/tests/causallyrelated.bench.ts +++ b/packages/object/tests/causallyrelated.bench.ts @@ -1,5 +1,5 @@ import { bench, describe } from "vitest"; -import { AddWinsSet } from "../../crdt/src/cros/AddWinsSet/index.js"; +import { AddWinsSet } from "../../blueprints/src/AddWinsSet/index.js"; import { type Hash, TopologyObject } from "../src/index.js"; describe("AreCausallyDependent benchmark", async () => { diff --git a/packages/object/benchmark/hashgraph.bench.ts b/packages/object/tests/hashgraph.bench.ts similarity index 86% rename from packages/object/benchmark/hashgraph.bench.ts rename to packages/object/tests/hashgraph.bench.ts index 6c902169..67f314f0 100644 --- a/packages/object/benchmark/hashgraph.bench.ts +++ b/packages/object/tests/hashgraph.bench.ts @@ -17,15 +17,15 @@ function benchmarkForAddWinSet( ); const cro = obj.cro as AddWinsSet; for (let j = 0; j < verticesPerCRO; j++) { - if (i % 3 === 2) { - cro.add(j); - cro.remove(j); - } else if (i % 3 === 1) { - cro.remove(j); - cro.add(j); - } else { - cro.add(j); - } + if (i % 3 === 2) { + cro.add(j); + cro.remove(j); + } else if (i % 3 === 1) { + cro.remove(j); + cro.add(j); + } else { + cro.add(j); + } } objects.push(obj); } diff --git a/packages/object/tsconfig.json b/packages/object/tsconfig.json index b53d3d8e..48e3023f 100644 --- a/packages/object/tsconfig.json +++ b/packages/object/tsconfig.json @@ -8,5 +8,5 @@ "path": "../logger" } ], - "include": ["src/**/*.ts", "benchmarks/**/*.ts"] + "include": ["src/**/*.ts"] } From b49fcca5dc0845600fa5a6f0d422e7647b7e99d2 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 18:07:58 +0700 Subject: [PATCH 3/9] update package.json --- packages/object/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/object/package.json b/packages/object/package.json index dbc69b4d..da9221c9 100644 --- a/packages/object/package.json +++ b/packages/object/package.json @@ -11,7 +11,6 @@ "files": [ "src", "dist", - "benchmark", "!dist/test", "!**/*.tsbuildinfo" ], @@ -33,7 +32,6 @@ "devDependencies": { "assemblyscript": "^0.27.29", "benchmark": "^2.1.4", - "ts-node": "^10.9.2", "tsx": "4.19.1" }, "dependencies": { From d4260cbe5133a505015a2637b30250e8bff6c179 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 21:04:19 +0700 Subject: [PATCH 4/9] feat: add github action --- .github/workflows/benchmark.yml | 36 ++++++++++++++++++++++++++++ packages/object/benchmark-output.txt | 4 ++++ packages/object/package.json | 2 +- pnpm-lock.yaml | 21 +++++++++++++++- 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 packages/object/benchmark-output.txt diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..1c180eb5 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,36 @@ +name: Benchmark.js Example +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run JavaScript benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - name: Run benchmark + run: cd packages/object && npm install && npm run benchmark + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: packages/object/benchmark-output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true \ No newline at end of file diff --git a/packages/object/benchmark-output.txt b/packages/object/benchmark-output.txt new file mode 100644 index 00000000..20c924c7 --- /dev/null +++ b/packages/object/benchmark-output.txt @@ -0,0 +1,4 @@ +Create HashGraph with 1000 vertices x 299 ops/sec ±6.75% (77 runs sampled) +Create 2 CROs (1000 vertices each) and Merge x 0.78 ops/sec ±50.48% (6 runs sampled) +Create 5 CROs (1000 vertices each) and Merge x 0.02 ops/sec ±6.59% (5 runs sampled) +Fastest is Create HashGraph with 1000 vertices diff --git a/packages/object/package.json b/packages/object/package.json index da9221c9..5c4ac459 100644 --- a/packages/object/package.json +++ b/packages/object/package.json @@ -27,7 +27,7 @@ "clean": "rm -rf dist/ node_modules/", "prepack": "tsc -b", "test": "vitest", - "benchmark-hashgraph": "npx tsx tests/hashgraph.bench.ts" + "benchmark": "npx tsx tests/hashgraph.bench.ts | tee benchmark-output.txt" }, "devDependencies": { "assemblyscript": "^0.27.29", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44e6ca9d..554c00eb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -409,7 +409,7 @@ importers: specifier: ^2.0.0 version: 2.2.1 '@topology-foundation/logger': - specifier: ^0.2.1-4 + specifier: ^0.2.1 version: link:../logger ts-proto: specifier: ^2.2.4 @@ -418,6 +418,12 @@ importers: assemblyscript: specifier: ^0.27.29 version: 0.27.30 + benchmark: + specifier: ^2.1.4 + version: 2.1.4 + tsx: + specifier: 4.19.1 + version: 4.19.1 packages: @@ -2451,6 +2457,9 @@ packages: before-after-hook@2.2.3: resolution: {integrity: sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==} + benchmark@2.1.4: + resolution: {integrity: sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ==} + binaryen@116.0.0-nightly.20240114: resolution: {integrity: sha512-0GZrojJnuhoe+hiwji7QFaL3tBlJoA+KFUN7ouYSDGZLSo9CKM8swQX8n/UcbR0d1VuZKU+nhogNzv423JEu5A==} hasBin: true @@ -4368,6 +4377,9 @@ packages: resolution: {integrity: sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA==} engines: {node: '>=10'} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + possible-typed-array-names@1.0.0: resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} engines: {node: '>= 0.4'} @@ -8037,6 +8049,11 @@ snapshots: before-after-hook@2.2.3: {} + benchmark@2.1.4: + dependencies: + lodash: 4.17.21 + platform: 1.3.6 + binaryen@116.0.0-nightly.20240114: {} bintrees@1.0.2: {} @@ -10219,6 +10236,8 @@ snapshots: dependencies: find-up: 5.0.0 + platform@1.3.6: {} + possible-typed-array-names@1.0.0: {} postcss@8.4.47: From 8f85728791582806311b30ba6de2e5c2001ccdc7 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 21:09:39 +0700 Subject: [PATCH 5/9] fix: update action setup node --- .github/workflows/benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1c180eb5..044fd769 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -17,8 +17,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 - cache: 'npm' + node-version: "20.x" + registry-url: "https://npm.pkg.github.com" - name: Run benchmark run: cd packages/object && npm install && npm run benchmark From 053c0ca6228c2a993a9644f9a31cfce6bbfa8507 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 21:21:54 +0700 Subject: [PATCH 6/9] fix: update action setup pnpm --- .github/workflows/benchmark.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 044fd769..a966cade 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,16 +11,18 @@ permissions: jobs: benchmark: - name: Run JavaScript benchmark + name: Run benchmark runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: pnpm/action-setup@v4 with: - node-version: "20.x" - registry-url: "https://npm.pkg.github.com" - - name: Run benchmark - run: cd packages/object && npm install && npm run benchmark + version: 9 + - shell: bash + run: | + pnpm install --no-frozen-lockfile + cd packages/object + pnpm run benchmark - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 From 00dc982ce4fcfe2e2ead5e98d697cafffbc2aeda Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 21:28:54 +0700 Subject: [PATCH 7/9] fix: remove too large test --- packages/object/tests/hashgraph.bench.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/object/tests/hashgraph.bench.ts b/packages/object/tests/hashgraph.bench.ts index 67f314f0..eaa77791 100644 --- a/packages/object/tests/hashgraph.bench.ts +++ b/packages/object/tests/hashgraph.bench.ts @@ -53,13 +53,6 @@ benchmarkForAddWinSet( true, ); -benchmarkForAddWinSet( - "Create 5 CROs (1000 vertices each) and Merge", - 5, - 1000, - true, -); - suite .on("cycle", (event) => { console.log(String(event.target)); From 12b90605222e850de0b87092c777da1a30f47bf9 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 21:52:25 +0700 Subject: [PATCH 8/9] trigger github action --- packages/object/benchmark-output.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/object/benchmark-output.txt b/packages/object/benchmark-output.txt index 20c924c7..f825cdd5 100644 --- a/packages/object/benchmark-output.txt +++ b/packages/object/benchmark-output.txt @@ -1,4 +1,3 @@ -Create HashGraph with 1000 vertices x 299 ops/sec ±6.75% (77 runs sampled) -Create 2 CROs (1000 vertices each) and Merge x 0.78 ops/sec ±50.48% (6 runs sampled) -Create 5 CROs (1000 vertices each) and Merge x 0.02 ops/sec ±6.59% (5 runs sampled) +Create HashGraph with 1000 vertices x 327 ops/sec ±2.17% (88 runs sampled) +Create 2 CROs (1000 vertices each) and Merge x 0.95 ops/sec ±20.86% (7 runs sampled) Fastest is Create HashGraph with 1000 vertices From 4dd675a5c4d1bc9a107837cc6958b818808cfe22 Mon Sep 17 00:00:00 2001 From: trungnotchung Date: Mon, 4 Nov 2024 22:19:24 +0700 Subject: [PATCH 9/9] remove output file --- packages/object/benchmark-output.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 packages/object/benchmark-output.txt diff --git a/packages/object/benchmark-output.txt b/packages/object/benchmark-output.txt deleted file mode 100644 index f825cdd5..00000000 --- a/packages/object/benchmark-output.txt +++ /dev/null @@ -1,3 +0,0 @@ -Create HashGraph with 1000 vertices x 327 ops/sec ±2.17% (88 runs sampled) -Create 2 CROs (1000 vertices each) and Merge x 0.95 ops/sec ±20.86% (7 runs sampled) -Fastest is Create HashGraph with 1000 vertices