diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 728717115..a9e7288cf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -26,16 +26,17 @@ jobs: --postgres-max-idle-conns=100 --experimental-features --otel-metrics-keep-in-memory & + - uses: extractions/setup-just@v2 - run: > - earthly - --allow-privileged - ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }} - ./test/performance+run --args="-benchtime 10s --ledger.url=http://localhost:3068 --parallelism=5" --locally=yes + just + --justfile ./test/performance/justfile + --working-directory ./test/performance + run . 5 10s 1 http://localhost:3068 - run: > - earthly - --allow-privileged - ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }} - ./test/performance+generate-graphs + just + --justfile ./test/performance/justfile + --working-directory ./test/performance + graphs - run: kill -9 $(ps aux | grep "ledger serve"| grep -v "grep" | awk '{print $2}') if: always() - uses: actions/upload-artifact@v4 diff --git a/flake.nix b/flake.nix index ce3863d9a..daa189554 100644 --- a/flake.nix +++ b/flake.nix @@ -101,6 +101,7 @@ just nodejs_22 self.packages.${system}.speakeasy + goperf ]; }; } diff --git a/test/performance/Earthfile b/test/performance/Earthfile deleted file mode 100644 index 20b0969fe..000000000 --- a/test/performance/Earthfile +++ /dev/null @@ -1,59 +0,0 @@ -VERSION 0.8 - -IMPORT github.com/formancehq/earthly:tags/v0.16.2 AS core - -run: - LOCALLY - ARG locally=no - ARG args="-bench=." - - IF [ $locally == "yes" ] - RUN rm -f ./report/benchmark-output.txt - RUN go test -run ^$ -tags it,local -report.file ./report/report.json -timeout 60m $args . | tee -a ./report/benchmark-output.txt - ELSE - FROM ../..+tidy - - ARG GOMAXPROCS=2 - ARG GOMEMLIMIT=1024MiB - - CACHE --id go-mod-cache /go/pkg/mod - CACHE --id go-cache /root/.cache/go-build - COPY *.go test/performance - COPY --dir scripts test/performance/ - - WORKDIR test/performance - RUN mkdir -p report - - WITH DOCKER --load=postgres:15-alpine=../../+postgres - RUN go test -run ^$ -tags it,local -report.file ./report/report.json -timeout 60m $args . | tee ./report/benchmark-output.txt - END - - SAVE ARTIFACT ./report/report.json - SAVE ARTIFACT ./report/benchmark-output.txt - END - -compare: - FROM core+builder-image - CACHE --id go-mod-cache /go/pkg/mod - CACHE --id go-cache /root/.cache/go-build - RUN go install golang.org/x/perf/cmd/benchstat@latest - ARG args="-bench=." - ARG rev=main - - COPY (+run/benchmark-output.txt --args=$args) /report/benchmark-output-local.txt - COPY --allow-privileged (github.com/formancehq/ledger/test/performance:${rev}+run/benchmark-output.txt --args=$args) /report/benchmark-output-remote.txt - - RUN benchstat /report/benchmark-output-remote.txt /report/benchmark-output-local.txt > benchmark-comparison.txt - - SAVE ARTIFACT benchmark-comparison.txt AS LOCAL benchmark-comparison.txt - -generate-graphs: - FROM core+base-image - RUN apk update && apk add nodejs npm - COPY charts /src - COPY ./report/report.json /report/report.json - WORKDIR /src - RUN npm install - RUN npm run build - RUN node index.js - SAVE ARTIFACT *.png AS LOCAL ./report/ diff --git a/test/performance/README.md b/test/performance/README.md index 8aef6bbaf..890624630 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -10,25 +10,25 @@ Scripts can be found in directory [scripts](./scripts). ## Run locally ```shell -earthly +run +just run ``` You can pass additional arguments (the underlying command is a standard `go test -bench=.`) using the flag `--args`. For example: ```shell -earthly +run --args="-benchtime 10s" +just run "-benchtime 10s" ``` ## Run on a remote stack ```shell -earthly +run --args="--stack.url=XXX --client.id=XXX --client.secret=XXX" +just run "--stack.url=XXX --client.id=XXX --client.secret=XXX" ``` ## Run on a remote ledger ```shell -earthly +run --args="--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX" +just run "--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX" ``` ## Results @@ -37,7 +37,7 @@ TPS is included as a benchmark metrics. You can generate some graphs using the command: ``` -earthly +generate-graphs +just graphs ``` See generated files in `report` directory. \ No newline at end of file diff --git a/test/performance/charts/index.js b/test/performance/charts/index.js index cf68fabac..c946da8fa 100644 --- a/test/performance/charts/index.js +++ b/test/performance/charts/index.js @@ -39,13 +39,14 @@ const main = () => __awaiter(void 0, void 0, void 0, function* () { let buffer = fs.readFileSync('../report/report.json', 'utf-8'); let reports = JSON.parse(buffer); yield (0, graphs_1.exportTPSGraph)({ - output: 'tps.png', + output: '../report/tps.png', }, reports); - yield (0, graphs_1.exportLatencyGraph)({ - output: 'p99.png' - }, 'P99', reports); - yield (0, graphs_1.exportLatencyGraph)({ - output: 'p95.png' - }, 'P95', reports); + yield (0, graphs_1.exportDatabaseStats)('../report/database_connections.png', reports); + const ps = ['P99', 'P95', 'P75', 'Avg']; + for (let p of ps) { + yield (0, graphs_1.exportLatencyGraph)({ + output: '../report/' + p.toLowerCase() + '.png' + }, p, reports); + } }); main(); diff --git a/test/performance/charts/index.ts b/test/performance/charts/index.ts index e5ac02f89..ba4a566f6 100644 --- a/test/performance/charts/index.ts +++ b/test/performance/charts/index.ts @@ -6,15 +6,15 @@ const main = async () => { let buffer = fs.readFileSync('../report/report.json', 'utf-8'); let reports = JSON.parse(buffer); await exportTPSGraph({ - output: 'tps.png', + output: '../report/tps.png', }, reports); - await exportDatabaseStats('database_connections.png', reports); + await exportDatabaseStats('../report/database_connections.png', reports); const ps: (keyof MetricsTime)[] = ['P99', 'P95', 'P75', 'Avg'] for (let p of ps) { await exportLatencyGraph({ - output: p.toLowerCase() + '.png' + output: '../report/' + p.toLowerCase() + '.png' }, p, reports); } } diff --git a/test/performance/charts/src/graphs.js b/test/performance/charts/src/graphs.js index a38670561..b85630917 100644 --- a/test/performance/charts/src/graphs.js +++ b/test/performance/charts/src/graphs.js @@ -12,9 +12,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.exportLatencyGraph = exports.exportTPSGraph = void 0; +exports.exportDatabaseStats = exports.exportLatencyGraph = exports.exportTPSGraph = void 0; const colors_1 = require("./colors"); const chartjs_to_image_1 = __importDefault(require("chartjs-to-image")); +const chart_js_1 = require("chart.js"); +const chartjs_plugin_annotation_1 = __importDefault(require("chartjs-plugin-annotation")); +chart_js_1.Chart.register(chartjs_plugin_annotation_1.default); const exportTPSGraph = (configuration, result) => __awaiter(void 0, void 0, void 0, function* () { const scripts = []; for (let script in result) { @@ -27,7 +30,7 @@ const exportTPSGraph = (configuration, result) => __awaiter(void 0, void 0, void const datasets = scripts.map(((script, index) => { return { label: script, - data: result[script].map(r => r.tps), + data: result[script].map(r => r.TPS), backgroundColor: colors_1.NAMED_COLORS[index % scripts.length], }; })); @@ -35,7 +38,7 @@ const exportTPSGraph = (configuration, result) => __awaiter(void 0, void 0, void type: 'bar', data: { labels: reportsForAnyScript - .map(r => r.configuration.name), + .map(r => r.Configuration.Name), datasets: datasets }, options: { @@ -76,7 +79,7 @@ const exportLatencyGraph = (configuration, key, result) => __awaiter(void 0, voi const datasets = scripts.map(((script, index) => { return { label: script, - data: result[script].map(r => r.metrics.Time[key].substring(0, r.metrics.Time[key].length - 2)), + data: result[script].map(r => parseFloat(r.Metrics.Time[key].substring(0, r.Metrics.Time[key].length - 2))), backgroundColor: colors_1.NAMED_COLORS[index % scripts.length], }; })); @@ -84,7 +87,7 @@ const exportLatencyGraph = (configuration, key, result) => __awaiter(void 0, voi type: 'bar', data: { labels: reportsForAnyScript - .map(r => r.configuration.name), + .map(r => r.Configuration.Name), datasets: datasets }, options: { @@ -112,3 +115,76 @@ const exportLatencyGraph = (configuration, key, result) => __awaiter(void 0, voi yield chart.toFile(configuration.output); }); exports.exportLatencyGraph = exportLatencyGraph; +const exportDatabaseStats = (output, result) => __awaiter(void 0, void 0, void 0, function* () { + const scope = 'github.com/uptrace/opentelemetry-go-extra/otelsql'; + const scripts = []; + for (let script in result) { + scripts.push(script); + } + const reportsForAnyScript = result[scripts[0]]; + if (!reportsForAnyScript) { + throw new Error("no data"); + } + const datasets = scripts.map(((script, index) => { + return { + label: script, + data: result[script].map(r => r.InternalMetrics.ScopeMetrics + .find(scopeMetric => scopeMetric.Scope.Name == scope) + .Metrics + .find(metric => metric.Name == 'go.sql.connections_open') + .Data + .DataPoints[0] + .Value), + backgroundColor: colors_1.NAMED_COLORS[index % scripts.length], + }; + })); + const maxConnection = reportsForAnyScript[0].InternalMetrics.ScopeMetrics + .find(scopeMetric => scopeMetric.Scope.Name == scope) + .Metrics + .find(metric => metric.Name == 'go.sql.connections_max_open') + .Data + .DataPoints[0] + .Value; + const config = { + type: 'bar', + data: { + labels: reportsForAnyScript.map(r => r.Configuration.Name), + datasets: datasets + }, + options: { + plugins: { + title: { + display: true, + text: 'Database connections' + }, + annotation: { + annotations: { + line1: { + type: 'line', + yMin: maxConnection, + yMax: maxConnection, + borderColor: 'rgb(255, 99, 132)', + borderWidth: 2, + } + } + } + }, + interaction: { + intersect: false, + }, + scales: { + x: { + stacked: false, + }, + y: { + stacked: false + } + } + } + }; + const chart = new chartjs_to_image_1.default(); + chart.setConfig(config); + chart.setChartJsVersion('4'); + yield chart.toFile(output); +}); +exports.exportDatabaseStats = exportDatabaseStats; diff --git a/test/performance/justfile b/test/performance/justfile new file mode 100755 index 000000000..464c38f5c --- /dev/null +++ b/test/performance/justfile @@ -0,0 +1,33 @@ +set dotenv-load +set positional-arguments + +tmpdir := `mktemp -d` + +run bench='.' p='1' benchtime='1s' count='1' ledger-url='' output='./report/benchmark-output.txt': + rm -f {{output}} + go test -run ^$ -tags it,local \ + -report.file ./report/report.json \ + -timeout 60m \ + -bench={{bench}} \ + -count={{count}} \ + --ledger.url={{ledger-url}} \ + -p {{p}} \ + -test.benchtime {{benchtime}} . | tee -a {{output}} + +compare bench='.' p='1' benchtime='1s' count='1' output='./report/benchmark-output.txt': + just run {{bench}} {{p}} {{benchtime}} {{count}} '' './report/benchmark-output-local.txt' + git clone --depth 1 -b main https://github.com/formancehq/ledger {{tmpdir}} + go test -run ^$ -tags it,local -report.file \ + ./report/report.json \ + -timeout 60m \ + -bench={{bench}} \ + -count={{count}} \ + -p {{p}} \ + -test.benchtime {{benchtime}} \ + {{tmpdir}}/test/performance | tee -a ./report/benchmark-output-main.txt + benchstat ./report/benchmark-output-main.txt ./report/benchmark-output-local.txt > ./report/benchmark-comparison.txt || true + +graphs: + cd charts && npm install + cd charts && npm run build + cd charts && node ./index.js \ No newline at end of file