forked from DOI-DO/j40-cejst-2
-
-
Notifications
You must be signed in to change notification settings - Fork 2
158 lines (157 loc) · 6.16 KB
/
pr_backend.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
name: Pull Request Backend
on:
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
env:
python-version: '3.10'
J40_VERSION_LABEL_STRING: ${{ vars.SCORE_VERSION }}
jobs:
# JOB to run change detection
detect-be-changes:
name: Detect backend changes
runs-on: ubuntu-latest
# Required permissions
permissions:
pull-requests: read
# Set job outputs to values from filter step
outputs:
backend: ${{ steps.filter.outputs.backend }}
steps:
# For pull requests it's not necessary to checkout the code
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
backend:
- 'data/**'
- '.github/workflows/pr_backend.yml'
code-quality-checks:
name: Code quality checks and tests
needs: detect-be-changes
if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: data/data-pipeline
environment: Staging
steps:
- name: Checkout source
uses: actions/checkout@v4
- name: Print variables to help debug
uses: hmarr/debug-action@v3
- name: Set up Python ${{ env.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Load cached Poetry installation
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: ~/.cache/pypoetry/virtualenvs
key: cejst-poetry-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
- name: Install poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: poetry install
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
- name: Check code is formatted
run: poetry run black --check data_pipeline/
- name: Check code style consistency
run: poetry run flake8 -v data_pipeline/
- name: Run static code analysis
run: poetry run pylint data_pipeline/
- name: Check library safety
run: poetry run safety check --ignore 51457 --ignore 44715 --ignore 70612 --ignore 74439
- name: Run unit tests
run: |
poetry run pytest data_pipeline/
generate-score-tiles:
name: Score and tile generation
needs: detect-be-changes
if: ${{ needs.detect-be-changes.outputs.backend == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: data/data-pipeline
environment: Staging
steps:
- name: Checkout source
uses: actions/checkout@v4
- name: Print variables to help debug
uses: hmarr/debug-action@v3
- name: Set up Python ${{ env.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.python-version }}
- name: Load cached Poetry installation
id: cached-poetry-dependencies
uses: actions/cache@v4
with:
path: ~/.cache/pypoetry/virtualenvs
key: cejst-poetry-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/deploy_backend_main.yml') }}
- name: Install poetry
uses: snok/install-poetry@v1
- name: Print Poetry settings
run: poetry show -v
- name: Install dependencies
run: poetry add s4cmd && poetry install
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
- name: Load cached ETL data
id: cached-etl-data
uses: actions/cache@v4
with:
path: |
data/data-pipeline/data_pipeline/data/census
data/data-pipeline/data_pipeline/data/dataset
key: cejst-dataset-env-${{ runner.os }}-${{ env.python-version }}-${{ hashFiles('data/data-pipeline/data_pipeline/etl/**/*') }}-${{ hashFiles('data/data-pipeline/data_pipeline/utils.py') }}
- name: Get Census Data
if: steps.cached-etl-data.outputs.cache-hit != 'true'
run: |
poetry run python3 -m data_pipeline.application census-data-download
- name: Run ETL
if: steps.cached-etl-data.outputs.cache-hit != 'true'
run: |
poetry run python3 -m data_pipeline.application etl-run
- name: Generate Score
run: |
poetry run python3 -m data_pipeline.application score-run
- name: Score Compare
run: |
poetry run python3 -m data_pipeline.comparator compare-score
- name: Generate Score Post
run: |
poetry run python3 -m data_pipeline.application generate-score-post
- name: Confirm we generated the version of the score we think we did
if: ${{ env.J40_VERSION_LABEL_STRING == '2.0' || env.J40_VERSION_LABEL_STRING == 'beta' }}
run: |
grep -v "Identified as disadvantaged due to tribal overlap" data_pipeline/data/score/downloadable/* > /dev/null
- name: Generate Score Geo
run: |
poetry run python3 -m data_pipeline.application geo-score
- name: Run smoketest for 1.0
if: ${{ env.J40_VERSION_LABEL_STRING == '1.0' }}
run: |
poetry run pytest data_pipeline/ -m smoketest
- name: Set timezone for tippecanoe
uses: szenius/[email protected]
with:
timezoneLinux: "America/Los_Angeles"
- name: Get tippecanoe
run: |
sudo apt-get install -y software-properties-common libsqlite3-dev zlib1g-dev
sudo apt-add-repository -y ppa:git-core/ppa
sudo mkdir -p /tmp/tippecanoe-src
sudo git clone https://github.com/mapbox/tippecanoe.git /tmp/tippecanoe-src
- name: Make tippecanoe
working-directory: /tmp/tippecanoe-src
run: |
sudo /usr/bin/bash -c make
mkdir -p /usr/local/bin
cp tippecanoe /usr/local/bin/tippecanoe
tippecanoe -v
- name: Generate Tiles
run: |
poetry run python3 -m data_pipeline.application generate-map-tiles
poetry run python3 -m data_pipeline.application generate-map-tiles --generate-tribal-layer