forked from rucio/rucio
-
Notifications
You must be signed in to change notification settings - Fork 0
152 lines (148 loc) · 7 KB
/
imagecache.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
name: Daily image cache build for autotests
on:
workflow_dispatch: {}
schedule:
- cron: '0 3 * * 1-5'
jobs:
setup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Identify branches
id: branches
run: |
BRANCHES="$(echo "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/branches{/branch}" | \
./tools/github/workflow/grabrelease.py --all --json --add master)"
echo "Will build image cache for branches: ${BRANCHES}"
echo "::set-output name=branches::${BRANCHES}"
outputs:
branches: ${{ steps.branches.outputs.branches }}
build_autotests:
runs-on: ubuntu-latest
needs:
- setup
strategy:
fail-fast: false
matrix:
branch: ${{ fromJson(needs.setup.outputs.branches) }}
steps:
- uses: actions/checkout@v2
with:
ref: ${{ matrix.branch }}
- name: Check and get files
id: files
shell: bash
run: |
MATRIX_PARSER_PY="${{ github.workspace }}/tools/test/matrix_parser.py"
MATRIX_CONF="${{ github.workspace }}/etc/docker/test/matrix.yml"
BUILD_IMAGES_PY="${{ github.workspace }}/tools/test/build_images.py"
echo "::set-output name=requirements_met::$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=matrix_parser::$MATRIX_PARSER_PY"
echo "::set-output name=matrix_configuration::$MATRIX_CONF"
echo "::set-output name=build_images::$BUILD_IMAGES_PY"
- name: Update pip, install python requirements for matrix parser
if: ${{ steps.files.outputs.requirements_met == 'true' }}
run: |
python3 -m pip install -U pip setuptools
python3 -m pip install -U PyYAML
- name: Identify Matrix
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: matrix
run: echo "::set-output name=matrix::$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})"
- name: Build and upload images
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: images
shell: bash
run: |
BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache")
if [[ "${{ github.event_name }}" == "schedule" ]]; then
# only build without cache on schedule (to update distro packages)
BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache")
fi
# the {} input for build_images.py is necessary until version 2
if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then
# new --branch option not available in older build_images.py revisions
BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}")
elif [[ "${{ matrix.branch }}" != "master" ]]; then
echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master"
exit 0
fi
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
i=0; until [ "$i" -ge 3 ]; do
IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} ./etc/docker/test || echo "")"
if [[ -n "$IMAGES" ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi
build_integration_tests:
runs-on: ubuntu-latest
needs:
- setup
strategy:
fail-fast: false
matrix:
branch: ${{ fromJson(needs.setup.outputs.branches) }}
steps:
- name: Checkout rucio containers repository
uses: actions/checkout@v2
with:
repository: rucio/containers
- name: Checkout rucio source
uses: actions/checkout@v2
with:
path: dev/rucio
ref: ${{ matrix.branch }}
- name: Check and get files
id: files
shell: bash
run: |
MATRIX_PARSER_PY="${{ github.workspace }}/dev/rucio/tools/test/matrix_parser.py"
MATRIX_CONF="${{ github.workspace }}/dev/rucio/etc/docker/test/matrix_integration_tests.yml"
BUILD_IMAGES_PY="${{ github.workspace }}/dev/rucio/tools/test/build_images.py"
echo "::set-output name=requirements_met::$(if [[ -r "$MATRIX_CONF" && -x "$MATRIX_PARSER_PY" && -x "$BUILD_IMAGES_PY" ]]; then echo "true"; else echo "false"; fi)"
echo "::set-output name=matrix_parser::$MATRIX_PARSER_PY"
echo "::set-output name=matrix_configuration::$MATRIX_CONF"
echo "::set-output name=build_images::$BUILD_IMAGES_PY"
- name: Update pip, install python requirements for matrix parser
if: ${{ steps.files.outputs.requirements_met == 'true' }}
run: |
python3 -m pip install -U pip setuptools
python3 -m pip install -U PyYAML
- name: Identify Matrix
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: matrix
run: echo "::set-output name=matrix::$(${{ steps.files.outputs.matrix_parser }} < ${{ steps.files.outputs.matrix_configuration }})"
- name: Build and upload images
if: ${{ steps.files.outputs.requirements_met == 'true' }}
id: images
shell: bash
run: |
BUILD_ARGS=("--output" "list" "--cache-repo" "ghcr.io/${{ github.repository }}" "--push-cache")
if [[ "${{ github.event_name }}" == "schedule" ]]; then
# only build without cache on schedule (to update distro packages)
BUILD_ARGS=("${BUILD_ARGS[@]}" "--build-no-cache")
fi
# the {} input for build_images.py is necessary until version 2
if echo '{}' | "${{ steps.files.outputs.build_images }}" --version-test 2; then
# new --branch option not available in older build_images.py revisions
BUILD_ARGS=("${BUILD_ARGS[@]}" "--branch" "${{ matrix.branch }}")
elif [[ "${{ matrix.branch }}" != "master" ]]; then
echo "::warning::Not building image for branch ${{ matrix.branch }}, because it would overwrite master"
exit 0
fi
docker login https://ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
i=0; until [ "$i" -ge 3 ]; do
IMAGES="$(echo '${{ steps.matrix.outputs.matrix }}' | "${{ steps.files.outputs.build_images }}" ${BUILD_ARGS[@]} \
${{ github.workspace }}/dev || echo "")"
if [[ -n "$IMAGES" ]]; then break;
else
i=$((i+1)); sleep 5;
echo "::warning::Building images failed, retrying…"
fi
done
docker logout https://ghcr.io
if [[ -z "$IMAGES" ]]; then echo "::error::Building images failed ultimately"; exit 1; fi