forked from siglens/siglens
-
Notifications
You must be signed in to change notification settings - Fork 0
166 lines (163 loc) · 6.37 KB
/
nightly-clickbench.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
name: Nightly-Clickbench-Test
on:
schedule:
- cron: '0 4 * * *' # Every day at 11PM EST (UTC-5)
workflow_dispatch: # Allows manual trigger as well
jobs:
Clickbench-Test:
timeout-minutes: 720 # 12 hrs
runs-on: [self-hosted, Linux, X64]
name: Clickbench
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.21
- name: Install dependencies
run: |
sudo yum install python3-pip
pip install requests
- uses: actions/checkout@v3
- name: Build Siglens
run: make build
- name: Run Siglens
run: |
./siglens --config server.yaml > output.log 2>&1 &
- name: Get Data Batch 1
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_1.tar
tar -xzvf splithits_1.tar
rm splithits_1.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_2.tar
tar -xzvf splithits_2.tar
rm splithits_2.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_3.tar
tar -xzvf splithits_3.tar
rm splithits_3.tar
- name: Send Data Batch 1
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_1 splithits_2 splithits_3
- name: Get Data Batch 2
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_4.tar
tar -xzvf splithits_4.tar
rm splithits_4.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_5.tar
tar -xzvf splithits_5.tar
rm splithits_5.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_6.tar
tar -xzvf splithits_6.tar
rm splithits_6.tar
- name: Send Data Batch 2
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_4 splithits_5 splithits_6
- name: Get Data Batch 3
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_7.tar
tar -xzvf splithits_7.tar
rm splithits_7.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_8.tar
tar -xzvf splithits_8.tar
rm splithits_8.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_9.tar
tar -xzvf splithits_9.tar
rm splithits_9.tar
- name: Send Data Batch 3
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_7 splithits_8 splithits_9
- name: Get Data Batch 4
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_10.tar
tar -xzvf splithits_10.tar
rm splithits_10.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_11.tar
tar -xzvf splithits_11.tar
rm splithits_11.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_12.tar
tar -xzvf splithits_12.tar
rm splithits_12.tar
- name: Send Data Batch 4
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_10 splithits_11 splithits_12
- name: Get Data Batch 5
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_13.tar
tar -xzvf splithits_13.tar
rm splithits_13.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_14.tar
tar -xzvf splithits_14.tar
rm splithits_14.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_15.tar
tar -xzvf splithits_15.tar
rm splithits_15.tar
- name: Send Data Batch 5
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_13 splithits_14 splithits_15
- name: Get Data Batch 6
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_16.tar
tar -xzvf splithits_16.tar
rm splithits_16.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_17.tar
tar -xzvf splithits_17.tar
rm splithits_17.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_18.tar
tar -xzvf splithits_18.tar
rm splithits_18.tar
- name: Send Data Batch 6
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_16 splithits_17 splithits_18
- name: Get Data Batch 7
run: |
cd tools/clickbench
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_19.tar
tar -xzvf splithits_19.tar
rm splithits_19.tar
wget https://github.com/siglens/pub-datasets/releases/download/5m-cb/splithits_20.tar
tar -xzvf splithits_20.tar
rm splithits_20.tar
- name: Send Data Batch 7
run: |
cd tools/clickbench
python3 send_datamulti.py
rm splithits_19 splithits_20
- name: Restart Siglens
run: |
pkill siglens
sleep 10
sed -i 's/pqsEnabled: true/pqsEnabled: false/' server.yaml
./siglens --config server.yaml > output2.log 2>&1 &
- name: Run Clickbench Test
run: |
sleep 5 # Wait for Siglens to start
cd tools/sigclient
go run main.go clickBench -d localhost:5122
- name: Print server execution logs for ingestion # For crashes
if: always()
run: |
cat output.log
- name: Print server execution logs for query # For crashes
if: always()
run: |
cat output2.log
- name: Print siglens logs
if: always()
run: |
cat logs/siglens.log