-
Notifications
You must be signed in to change notification settings - Fork 0
156 lines (152 loc) · 6.24 KB
/
cicd.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: CI/CD Builds
on:
push:
branches:
- "**"
paths-ignore:
- docs/**.md
jobs:
python_build_and_test:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version:
# - 3.7
- 3.8
steps:
- name: Clone git repo
uses: actions/checkout@v1
- name: "Print Version Info (build number ${{ github.run_number }})"
run: |
echo "Build Number: $GITHUB_RUN_NUMBER"
echo "Version: $(cat VERSION).$GITHUB_RUN_NUMBER"
echo "Run ID: ${{ github.run_id }}"
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install pipx and poetry
run: |
python3 -m pip install pipx
python3 -m pipx ensurepath
pipx install poetry
- name: Install tapdance
run: |
poetry install
# - name: Lint with flake8
# run: |
# # stop the build if there are Python syntax errors or undefined names
# poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
# #poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
pypi_publish:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
python-version:
- 3.7
needs:
- python_build_and_test
steps:
- name: Clone git repo
uses: actions/checkout@v1
- name: Set up Python (v${{ matrix.python-version }})
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Install pipx and poetry
run: |
python3 -m pip install pipx
python3 -m pipx ensurepath
pipx install poetry
- name: Push to PyPi (https://pypi.org/project/tapdance/) (build number ${{ github.run_number }})
env:
BRANCH_NAME: ${{ github.ref }}
PYPI_USER: ${{ secrets.PYPI_USER }}
PYPI_PASS: ${{ secrets.PYPI_SECRET }}
run: |
if [[ "$BRANCH_NAME" == *master ]]
then
poetry version --short
else
poetry version $(poetry version --short)-dev.$GITHUB_RUN_NUMBER
poetry version --short
fi
echo -e "\nPublishing to version ref '$(poetry version --short)'...\n\n"
poetry config http-basic.pypi $PYPI_USER $PYPI_PASS
poetry publish --build
- name: Wait up to 3 minutes for PyPi availability
run: |
pwd
ls -la
export VER=$(cat VERSION)
export PIPERR=$(pip install tapdance==$VER 2>&1)
echo "Checking for PyPi availability of version $VER"
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not yet found..."; sleep 30; } fi;
export PIPERR=$(pip install tapdance==$VER 2>&1)
if [[ $PIPERR == *"$VER"* ]]; then { echo "Yes"; } else { echo "Not found. Giving up. Last message from PyPi was $PIPERR"; exit 1; } fi;
docker_builds:
runs-on: ubuntu-latest
needs:
- pypi_publish
steps:
- name: Clone git repo
uses: actions/checkout@v1
- name: Initialize 'pre' flag if not on main branch
run: |
if [[ "${{ github.ref }}" == *main ]]
then
echo "PRE_FLAG=" >> $GITHUB_ENV
elif [[ "${{ github.ref }}" == *master ]]
then
echo "PRE_FLAG=" >> $GITHUB_ENV
else
echo "PRE_FLAG=--pre" >> $GITHUB_ENV
fi
- name: Print version, branch, and prerelease flag
run: |
echo "Build Number: $GITHUB_RUN_NUMBER"
echo "Version: $(cat VERSION).$GITHUB_RUN_NUMBER"
echo "Branch Name: ${{ github.ref }}"
echo "PRE_FLAG: $PRE_FLAG"
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: 3.7
- name: Install pipx and poetry
run: |
python3 -m pip install pipx
python3 -m pipx ensurepath
pipx install poetry
- name: Install tapdance
run: poetry install
- name: Login to docker
run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_SECRET }}
- name: Build and push all *prioritized* singer images (build number ${{ github.run_number }})
run: |
cd docker
poetry run tapdance build_image sparkthink snowflake-test --push $PRE_FLAG
poetry run tapdance build_image dbt snowflake-test --push $PRE_FLAG
poetry run tapdance build_image powerbi-metadata snowflake-test --push $PRE_FLAG
poetry run tapdance build_image mssql snowflake-test --push $PRE_FLAG
# poetry run tapdance build_image pardot snowflake-test --push $PRE_FLAG
# poetry run tapdance build_image salesforce snowflake-test --push $PRE_FLAG
# poetry run tapdance build_image sftp snowflake-test --push $PRE_FLAG
# poetry run tapdance build_image covid-19 s3-csv --push $PRE_FLAG
# poetry run tapdance build_image covid-19 s3-csv-test --push $PRE_FLAG
- name: Build and push all singer images
run: cd docker && poetry run tapdance build_all_images --push $PRE_FLAG
- name: Print list of images
run: docker image ls