-
Notifications
You must be signed in to change notification settings - Fork 14
243 lines (207 loc) · 8.79 KB
/
main-build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
name: Build
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name}}-build
cancel-in-progress: true
on:
workflow_dispatch:
inputs:
push_containers:
type: boolean
required: false
default: false
description: push the containers to the registry
target_tag:
type: string
required: false
default: main_build
description: Version tag of existing containers to use. Is `main_build` by default.
deploy_to_viash_hub:
type: boolean
required: false
default: false
description: Also build packages and docker images for viash-hub.com and push them.
push:
branches: [ 'main' ]
jobs:
# phase 1
build_and_deploy_target_folder:
runs-on: ubuntu-latest
outputs:
component_matrix: ${{ steps.set_matrix.outputs.matrix }}
cache_key: ${{ steps.cache.outputs.cache_key }}
steps:
- name: Keep symlinks as-is
run: |
git config --global core.symlinks true
- name: Branch to checkout (use existing target branch if it exists)
id: get_checkout_branch
run: |
if ! git ls-remote --heads --exit-code https://github.com/openpipelines-bio/openpipeline.git ${{ github.ref_name }}_build > /dev/null; then
echo "Remote branch does not exist, fetching current branch and building on top of it"
echo "checkout_branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
else
echo "Remote branch exists, checking out existing branch"
echo "checkout_branch=${{ github.ref_name }}_build" >> "$GITHUB_OUTPUT"
fi
- uses: actions/checkout@v4
with:
ref: ${{ steps.get_checkout_branch.outputs.checkout_branch }}
fetch-depth: 0
- name: Fetch changes from main
run: |
git fetch origin ${{github.ref_name}}
git checkout -f --no-overlay origin/${{github.ref_name}} -- '.'
- uses: viash-io/viash-actions/setup@v4
- name: Remove target folder from .gitignore
run: |
# allow publishing the target folder
sed -i '/^\/target\/$/d' .gitignore
- uses: viash-io/viash-actions/ns-build@v4
with:
config_mod: |
.functionality.version := '${{ github.ref_name }}_build'
.platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}'
parallel: true
query: ^(?!workflows)
- uses: viash-io/viash-actions/ns-build@v4
with:
config_mod: .functionality.version := '${{ github.ref_name }}_build'
parallel: true
query: ^workflows
- name: Build nextflow schemas
uses: viash-io/viash-actions/pro/build-nextflow-schemas@v4
with:
components: src
workflows: src
viash_pro_token: ${{ secrets.GTHB_PAT }}
tools_version: 'main_build'
- name: Build parameter files
uses: viash-io/viash-actions/pro/build-nextflow-params@v4
with:
workflows: src
components: src
viash_pro_token: ${{ secrets.GTHB_PAT }}
tools_version: 'main_build'
- name: Deploy to target branch
uses: stefanzweifel/git-auto-commit-action@v5
with:
create_branch: true
commit_message: "deploy: ${{github.sha}}"
skip_dirty_check: true
branch: ${{ github.ref_name }}_build
- id: ns_list
uses: viash-io/viash-actions/ns-list@v4
with:
platform: docker
src: src
format: json
query_namespace: ^(?!workflows)
- id: set_matrix
run: |
echo "matrix=$(jq -c '[ .[] |
{
"name": (.functionality.namespace + "/" + .functionality.name),
"config": .info.config,
"dir": .info.config | capture("^(?<dir>.*\/)").dir
}
]' ${{ steps.ns_list.outputs.output_file }} )" >> $GITHUB_OUTPUT
- uses: viash-io/viash-actions/ns-build@v4
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
with:
config_mod: |
.functionality.version := '${{ github.ref_name }}_build'
.platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}'
.platforms[.type == 'docker'].target_organization := 'openpipelines-bio/openpipeline'
.platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
.platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/openpipelines-bio/openpipeline'
parallel: true
query: ^(?!workflows)
- uses: viash-io/viash-actions/ns-build@v4
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
with:
config_mod: |
.functionality.version := '${{ github.ref_name }}_build'
parallel: true
query: ^workflows
- name: Push source branch to Viash-hub
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
run: |
git push -f https://x-access-token:${{ secrets.VIASHHUB_PAT }}@viash-hub.com/openpipelines-bio/openpipeline.git -u ${{ github.ref_name }} ${{ github.ref_name }}
- name: Set origin to viash-hub
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
# This is needed because git-auto-commit-action uses origin by default
run: git remote set-url origin https://x-access-token:${{ secrets.VIASHHUB_PAT }}@viash-hub.com/openpipelines-bio/openpipeline.git
- name: Deploy to target branch
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
create_branch: true
commit_message: "deploy: ${{github.sha}}"
skip_dirty_check: true
branch: ${{ github.ref_name }}_build
# phase 2
build_and_deploy_docker_containers:
needs: build_and_deploy_target_folder
if: ${{github.event_name == 'push' || github.event.inputs.push_containers == 'true' }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
component: ${{ fromJson(needs.list.outputs.component_matrix) }}
steps:
# Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
- uses: data-intuitive/reclaim-the-bytes@v2
- uses: actions/checkout@v4
- uses: viash-io/viash-actions/setup@v4
- name: Build container
uses: viash-io/viash-actions/ns-build@v4
with:
config_mod: |
.functionality.version := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}'
.platforms[.type == 'docker'].target_tag := '${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }}'
platform: docker
src: ${{ matrix.component.dir }}
setup: build
- name: Login to container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GTHB_USER }}
password: ${{ secrets.GTHB_PAT }}
- name: Push container
uses: viash-io/viash-actions/ns-build@v4
with:
config_mod: .functionality.version := '${{ github.ref_name }}_build'
platform: docker
src: ${{ matrix.component.dir }}
setup: push
- name: Login to Viash-Hub container registry
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
uses: docker/login-action@v3
with:
registry: viash-hub.com:5050
username: ${{ secrets.VIASHHUB_USER }}
password: ${{ secrets.VIASHHUB_PAT }}
- name: Update Docker settings
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
run: |
sudo sed -i 's/ }/, \"max-concurrent-uploads\": 2 }/' /etc/docker/daemon.json
sudo systemctl restart docker
- name: "Re-tag containers for viash-hub"
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
run: |
viash ns exec --apply_platform -p docker \
'docker tag ghcr.io/openpipelines-bio/{namespace}_{functionality-name}:${{ github.event_name == 'push' && 'main_build' || github.event.inputs.target_tag }} \
viash-hub.com:5050/openpipelines-bio/openpipeline/{namespace}_{functionality-name}:${{ github.ref_name }}_build'
- name: Push container to Viash-Hub
if: ${{ github.event.inputs.deploy_to_viash_hub == 'true' }}
uses: viash-io/viash-actions/ns-build@v4
with:
config_mod: |
.functionality.version := '${{ github.ref_name }}_build'
.platforms[.type == 'docker'].target_registry := 'viash-hub.com:5050'
.platforms[.type == 'docker'].target_organization := 'openpipelines-bio/openpipeline'
.platforms[.type == 'docker'].target_image_source := 'https://viash-hub.com/openpipelines-bio/openpipeline'
platform: docker
src: ${{ matrix.component.dir }}
setup: push