Skip to content

Commit a62d86b

Browse files
wuputahwuputah
authored
more automated benchmarking improvements (#105)
* move most of the benchmark logic into a script * remove `if` on benchmarks action
1 parent 7905659 commit a62d86b

File tree

2 files changed

+43
-92
lines changed

2 files changed

+43
-92
lines changed

‎.github/workflows/benchmark.yaml

+19-87
Original file line numberDiff line numberDiff line change
@@ -35,120 +35,52 @@ on:
3535
- warehouse-10G
3636
tag:
3737
type: string
38-
3938
env:
4039
BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }}
4140
REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }}
42-
BASE_IMAGE: ${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
4341
TAG: ${{ inputs.tag || format('15-{0}', github.sha) }}
44-
NAME: hydra-benchmark
45-
BENCHER_PROJECT: hydra-${{ (inputs.repo == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' && 'spilo') || 'postgres' }}
4642
BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
47-
BENCHER_ADAPTER: json
4843
BENCHER_TESTBED: gh-4core
4944

5045
jobs:
5146
benchmarks:
5247

5348
name: Run Benchmark
54-
if: github.repository == 'hydradatabase/hydra-internal'
5549
# 16gb ram, 4vcpu, 150gb disk
5650
runs-on: benchmarks-ubuntu-latest-4core
5751

5852
steps:
59-
60-
- name: Configure AWS credentials
61-
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
62-
uses: aws-actions/configure-aws-credentials@v1
63-
with:
64-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
65-
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
66-
aws-region: us-east-1
67-
mask-aws-account-id: no
68-
69-
- name: Login to Amazon ECR
70-
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
71-
id: spilo-ecr
72-
uses: aws-actions/amazon-ecr-login@v1
73-
with:
74-
registries: "011789831835"
75-
7653
- uses: actions/setup-node@v3
7754
with:
7855
node-version: 18
7956

8057
- uses: docker/setup-buildx-action@v2
8158

59+
- uses: unfor19/install-aws-cli-action@v1
60+
with:
61+
arch: amd64
62+
63+
- uses: bencherdev/bencher@main
64+
8265
- name: Checkout benchmarks
8366
uses: actions/checkout@v3
8467
with:
8568
repository: hydradatabase/benchmarks
8669

87-
- name: Set up docker image
88-
run: |
89-
docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -v $PWD:/benchmarks -m 12288m --cpus=4 --shm-size=1024m --name=$NAME $REPO:$TAG
90-
91-
- name: Prepare to download data
92-
run: |
93-
set -eux
94-
BENCHMARK_SRC="$(echo $BENCHMARK | cut -f 1 -d -)"
95-
if [ "$BENCHMARK" != "$BENCHMARK_SRC" ]; then
96-
ln -s $BENCHMARK_SRC $BENCHMARK
97-
fi
98-
mkdir -p $BENCHMARK/data
99-
tree $BENCHMARK
100-
101-
- name: Download data
102-
uses: keithweaver/[email protected]
70+
- name: Configure AWS credentials
71+
uses: aws-actions/configure-aws-credentials@v1
10372
with:
104-
command: cp
105-
source: s3://hydra-benchmarks/data/${{ env.BENCHMARK }}
106-
destination: ./${{ env.BENCHMARK }}/data
107-
aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
108-
aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
109-
aws_region: us-east-1
110-
flags: --no-progress --recursive
111-
112-
- name: Prepare data
113-
run: |
114-
set -eux
115-
pushd $BENCHMARK/data
116-
for f in *.gz; do
117-
TARGET="$(basename $f .gz)"
118-
mkfifo $TARGET
119-
nohup gzip -d -c $f >$TARGET &
120-
done
121-
popd
122-
tree $BENCHMARK
123-
124-
- name: Run benchmark
125-
run: |
126-
docker exec $NAME /bin/sh -c "RUNTIME=now /benchmarks/run-benchmark.sh -z -b $BENCHMARK -v zstd -u postgres"
127-
./analyze.js ./results/$BENCHMARK/zstd/now > ./results.json
128-
129-
- name: Cleanup docker image
130-
run: |
131-
docker stop $NAME
132-
docker rm $NAME
73+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
74+
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
75+
aws-region: us-east-1
76+
mask-aws-account-id: no
13377

134-
- name: Upload result to S3
135-
uses: keithweaver/[email protected]
78+
- name: Login to Amazon ECR
79+
if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
80+
id: spilo-ecr
81+
uses: aws-actions/amazon-ecr-login@v1
13682
with:
137-
command: cp
138-
source: ./results.json
139-
destination: s3://hydra-benchmarks/results/${{ env.BASE_IMAGE }}-${{ env.BENCHMARK }}-zstd-${{ env.TAG }}.json
140-
aws_access_key_id: ${{ secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
141-
aws_secret_access_key: ${{ secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
142-
aws_region: us-east-1
143-
flags: --no-progress
144-
145-
- uses: bencherdev/bencher@main
83+
registries: "011789831835"
14684

147-
- name: Upload result to bencher
148-
run: |
149-
bencher run \
150-
--if-branch "$GITHUB_REF_NAME" \
151-
--else-if-branch "$GITHUB_BASE_REF" \
152-
--else-if-branch main \
153-
--err \
154-
"./analyze-bencher.js ./results/$BENCHMARK/zstd/now $BENCHMARK"
85+
- name: Run benchmark
86+
run: ./run-gha.sh

‎.github/workflows/ci.yaml

+24-5
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ jobs:
3636
run: make lint_acceptance
3737

3838
validate_columnar:
39+
name: Validate Columnar ${{ matrix.postgres }}
3940
strategy:
4041
fail-fast: false
4142
matrix:
4243
postgres: ["13", "14", "15"]
43-
name: Validate Columnar ${{ matrix.postgres }}
4444
runs-on: ubuntu-latest
4545
timeout-minutes: 15
4646
steps:
@@ -62,12 +62,12 @@ jobs:
6262
columnar.target=checker
6363
6464
build_validate_postgres:
65+
name: Build and Validate Postgres ${{ matrix.postgres }}
6566
needs: [validate_columnar]
6667
strategy:
6768
fail-fast: false
6869
matrix:
6970
postgres: ["13", "14", "15"]
70-
name: Build and Validate Postgres ${{ matrix.postgres }}
7171
runs-on: ubuntu-latest
7272
timeout-minutes: 15
7373
steps:
@@ -121,9 +121,9 @@ jobs:
121121
targets: postgres
122122

123123
build_validate_spilo:
124+
name: Build and Validate Spilo
124125
needs: [validate_columnar]
125126
if: github.repository == 'hydradatabase/hydra-internal'
126-
name: Build and Validate Spilo
127127
runs-on: ubuntu-latest
128128
timeout-minutes: 15
129129
steps:
@@ -196,13 +196,13 @@ jobs:
196196
*.platform=linux/amd64
197197
198198
push_postgres:
199+
name: Push Postgres ${{ matrix.postgres }}
199200
needs: [build_validate_postgres]
200201
if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
201202
strategy:
202203
fail-fast: false
203204
matrix:
204205
postgres: ["13", "14", "15"]
205-
name: Push Postgres ${{ matrix.postgres }}
206206
runs-on: ubuntu-latest
207207
timeout-minutes: 15
208208
steps:
@@ -231,6 +231,7 @@ jobs:
231231
${{ matrix.postgres == '14' && format('postgres.tags={0}:latest', env.POSTGRES_REPO) || '' }}
232232
233233
push_spilo:
234+
name: Push Spilo
234235
needs: [build_validate_spilo]
235236
if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
236237
uses: ./.github/workflows/build_spilo.yaml
@@ -240,13 +241,31 @@ jobs:
240241
secrets: inherit
241242

242243
benchmark_postgres:
244+
name: Benchmark Hydra Postgres image (${{ matrix.benchmark }})
243245
needs: [push_postgres]
244246
if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
245247
strategy:
248+
fail-fast: false
246249
matrix:
247250
benchmark: ["clickbench-100M", "warehouse-10G"]
248251
uses: ./.github/workflows/benchmark.yaml
249252
with:
250253
benchmark: ${{ matrix.benchmark }}
251254
repo: ghcr.io/hydradatabase/hydra
252-
tag: 15-${{ github.sha }}
255+
tag: 14-${{ github.sha }}
256+
secrets: inherit
257+
258+
benchmark_spilo:
259+
name: Benchmark Hydra Spilo image (${{ matrix.benchmark }})
260+
needs: [push_spilo]
261+
if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
262+
strategy:
263+
fail-fast: false
264+
matrix:
265+
benchmark: ["clickbench-100M", "warehouse-10G"]
266+
uses: ./.github/workflows/benchmark.yaml
267+
with:
268+
benchmark: ${{ matrix.benchmark }}
269+
repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
270+
tag: ${{ github.sha }}
271+
secrets: inherit

0 commit comments

Comments
 (0)