-
Notifications
You must be signed in to change notification settings - Fork 11
198 lines (193 loc) · 7.09 KB
/
scale-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
---
# yamllint disable rule:truthy rule:truthy rule:line-length
name: "Scale testing"
on:
schedule:
- cron: "0 2 * * *"
workflow_dispatch:
inputs:
commit:
type: string
description: commit sha or branch name
default: ''
required: false
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
INFRAHUB_DB_USERNAME: neo4j
INFRAHUB_DB_PASSWORD: admin
INFRAHUB_DB_ADDRESS: localhost
INFRAHUB_DB_PORT: 7687
INFRAHUB_DB_PROTOCOL: bolt
INFRAHUB_BROKER_ADDRESS: message-queue
INFRAHUB_LOG_LEVEL: CRITICAL
INFRAHUB_IMAGE_NAME: "opsmill/infrahub"
INFRAHUB_IMAGE_VER: "local"
jobs:
scale-tests:
strategy:
fail-fast: false
max-parallel: 4
matrix:
include:
- name: 1000 nodes
node-amount: 1000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_infranode
- name: 10000 nodes
node-amount: 10000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_infranode
- name: 100000 nodes
node-amount: 100000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_infranode
- name: 100 attrs
node-amount: 10
attrs-amount: 100
rels-amount: 0
changes-amount: 0
stager: stage_infranode
- name: 1000 attrs
node-amount: 10
attrs-amount: 1000
rels-amount: 0
changes-amount: 0
stager: stage_infranode
- name: 100 rels
node-amount: 10
attrs-amount: 0
rels-amount: 100
changes-amount: 0
stager: stage_infranode
- name: 1000 rels
node-amount: 10
attrs-amount: 0
rels-amount: 1000
changes-amount: 0
stager: stage_infranode
- name: 100 branches (with new nodes)
node-amount: 100
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch
- name: 100 branches (with single node)
node-amount: 100
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch_update
- name: 1000 branches (with new nodes)
node-amount: 1000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch
- name: 1000 branches (with single node)
node-amount: 1000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch_update
- name: 1000 updates of a single node
node-amount: 1
attrs-amount: 0
rels-amount: 0
changes-amount: 1000
stager: stage_infranode
- name: 100 updates of a single node
node-amount: 1
attrs-amount: 0
rels-amount: 0
changes-amount: 100
stager: stage_infranode
- name: 100 updates of 100 nodes
node-amount: 100
attrs-amount: 0
rels-amount: 0
changes-amount: 100
stager: stage_infranode
- name: 1000 changes in diff
node-amount: 1000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch_diff
- name: 10000 changes in diff
node-amount: 10000
attrs-amount: 0
rels-amount: 0
changes-amount: 0
stager: stage_branch_diff
name: ${{ matrix.name }}
runs-on:
group: huge-runners
timeout-minutes: 300
env:
INFRAHUB_DB_TYPE: neo4j
METRICS_ENDPOINT: ${{ secrets.METRICS_ENDPOINT }}
steps:
- name: "Check out repository code"
uses: "actions/checkout@v4"
with:
ref: ${{ inputs.commit == '' && 'develop' || inputs.commit }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: "Setup python environment"
run: |
pipx install poetry
poetry config virtualenvs.prefer-active-python true
pip install invoke toml
- name: "Install Package"
run: "poetry install --with test-scale"
- name: "Set environment variables"
run: echo INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }} >> $GITHUB_ENV
- name: "Set environment variables"
run: echo INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }} >> $GITHUB_ENV
- name: "Clear docker environment"
run: docker compose -p $INFRAHUB_BUILD_NAME down -v --remove-orphans --rmi local
- name: Select infrahub port
run: echo "INFRAHUB_SERVER_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
- name: Set INFRAHUB_URL
run: echo "INFRAHUB_URL=http://localhost:${INFRAHUB_SERVER_PORT}" >> $GITHUB_ENV
- name: Select infrahub db port
run: echo "INFRAHUB_DB_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
- name: Select infrahub db port
run: echo "INFRAHUB_DB_BACKUP_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
- name: Select vmagent port
run: echo "VMAGENT_PORT=$(shuf -n 1 -i 10000-30000)" >> $GITHUB_ENV
- name: Set job name
run: echo "JOB_NAME=${{ matrix.name }}" >> $GITHUB_ENV
- name: "Start scale environment"
run: "poetry run invoke backend.test-scale-env-start -g 1"
- name: "Store start time"
run: echo TEST_START_TIME=$(date +%s)000 >> $GITHUB_ENV
- name: "Run test"
run: "poetry run invoke backend.test-scale --amount ${{ matrix.node-amount }} --attrs ${{ matrix.attrs-amount }} --rels ${{ matrix.rels-amount }} --changes ${{ matrix.changes-amount }} -s backend/tests/scale/schema.yml --stager ${{ matrix.stager }}"
- name: "Push metrics"
if: always()
run: 'curl -v --data-binary @metrics.csv "http://localhost:${VMAGENT_PORT}/api/v1/import/csv?format=1:metric:scale_test_response_time,2:time:unix_ms,3:label:operation_name,4:label:stage,5:label:node_amount,6:label:attrs_amount,7:label:rels_amount"'
- name: "Wait for vmagent to push metrics"
if: always()
run: "sleep 10"
- name: "Show graph URL"
if: always()
run: 'echo "https://grafana-prod.tailc018d.ts.net/d/ebf7ec72-db79-4fb7-9b46-4621ca9c407a/scale-tests?orgId=1&var-run_id=$GITHUB_RUN_ID&var-job=${JOB_NAME// /%20}&var-stage=test&var-node_amount=${{ matrix.node-amount }}&var-attrs_amount=${{ matrix.attrs-amount }}&var-rels_amount=${{ matrix.rels-amount }}&var-runner=$INFRAHUB_BUILD_NAME&from=$TEST_START_TIME&to=$(date +%s)000"'
- name: Display server logs
if: always()
run: docker logs "${INFRAHUB_BUILD_NAME}-infrahub-server-1"
- name: "Destroy scale environment"
if: always()
run: "invoke backend.test-scale-env-destroy"
# - name: "Print results"
# run: "cat *.csv"