-
Notifications
You must be signed in to change notification settings - Fork 9
262 lines (258 loc) · 12.2 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
name: unifrac-binaries CI
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
build-and-test:
strategy:
matrix:
os: [ubuntu-latest, macos-13, macos-14, linux-gpu-cuda]
offload: [cpu, acc, ompgpu]
exclude:
- os: linux-gpu-cuda
offload: cpu
- os: macos-14
offload: acc
- os: macos-14
offload: ompgpu
- os: macos-13
offload: acc
- os: macos-13
offload: ompgpu
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: conda-incubator/setup-miniconda@v3
with:
miniconda-version: "latest"
auto-update-conda: true
- name: Install
shell: bash -l {0}
run: |
df -h .
if [[ "$(uname -s)" == "Linux" ]];
then
conda create -q --yes --strict-channel-priority -n unifrac -c conda-forge -c bioconda gxx_linux-64 gfortran_linux-64 hdf5 lz4 zlib hdf5-static libcblas liblapacke blas-devel make curl
else
if [[ "x`conda info |awk '/platform/{print $3}'`" == "xosx-64" ]];
then
conda create -q --yes --strict-channel-priority -n unifrac -c conda-forge -c bioconda clangxx_osx-64 hdf5 lz4 hdf5-static libcblas liblapacke blas-devel make curl
else
conda create -q --yes --strict-channel-priority -n unifrac -c conda-forge -c bioconda clangxx_osx-arm64 hdf5 lz4 hdf5-static libcblas liblapacke blas-devel make curl
fi
fi
conda clean --yes -t
df -h .
conda activate unifrac
echo "$(uname -s)"
if [[ "$(uname -s)" == "Linux" ]];
then
which x86_64-conda-linux-gnu-gcc
x86_64-conda-linux-gnu-gcc -v
x86_64-conda-linux-gnu-g++ -v
else
which clang
clang -v
fi
which h5c++
export PERFORMING_CONDA_BUILD=True
export BUILD_NV_OFFLOAD=${{ matrix.offload }}
if [[ "$(uname -s)" == "Linux" ]] && [[ "x${BUILD_NV_OFFLOAD}" != "xcpu" ]];
then
# install PGI locally
./scripts/install_hpc_sdk.sh </dev/null
# also install the AMD compiler
./scripts/install_amd_clang.sh </dev/null
# get the compilers in the path and set NV_CXX and AMD_CXX
. ./setup_nv_compiler.sh
. ./setup_amd_compiler.sh
fi
# else, no NV_CXX or AMD_CXX in the env, so no GPU build
df -h .
echo "======= begin env ====="
env
echo "======= end env ====="
# all == build (shlib,bins,tests) and install
make all
df -h .
- name: Tests
shell: bash -l {0}
run: |
conda activate unifrac
if [[ "$(uname -s)" == "Linux" ]];
then
lscpu
fi
# keep it low for runs in containers
# and a weird number to potentially catch potential bugs
export OMP_NUM_THREADS=3
# diagnostic messages for debugging, if needed
export UNIFRAC_CPU_INFO=Y
export UNIFRAC_GPU_INFO=Y
pushd src
./test_su
./test_api
./test_ska
popd
pushd test
./capi_test 1
./capi_inmem_test 1
# explicitly check that we do not have hdf5 dependencies
if [[ "$(uname -s)" == "Linux" ]];
then
ldd ./capi_test |awk 'BEGIN{a=0}/hdf/{a=a+1;print $0}END{if (a==0) {print "No dynamic hdf5 found"} else {exit 2}}'
else
otool -L ./capi_test|awk 'BEGIN{a=0}/hdf/{a=a+1;print $0}END{if (a==0) {print "No dynamic hdf5 found"} else {exit 2}}'
fi
popd
pushd src/testdata
conda install --yes -c conda-forge h5py python=3.10
# weighted_unnormalized
time ssu -m weighted_unnormalized_fp32 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 -o t1.h5
./compare_unifrac_matrix.py test500.weighted_unnormalized_fp32.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.weighted_unnormalized_fp32.h5 t1.h5 3 0.1
rm -f t1.h5
# retry with default precision handling
time ssu -m weighted_unnormalized -i test500.biom -t test500.tre --pcoa 4 -r hdf5 -o t1.h5
./compare_unifrac_matrix.py test500.weighted_unnormalized_fp32.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.weighted_unnormalized_fp32.h5 t1.h5 3 0.1
rm -f t1.h5
time ssu -f -m weighted_unnormalized_fp32 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 -o t1.h5
# matrrix will be different, but PCOA similar
./compare_unifrac_pcoa.py test500.weighted_unnormalized_fp32.h5 t1.h5 3 0.1
rm -f t1.h5
time ssu -m weighted_unnormalized_fp64 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp64 -o t1.h5
# minimal precision loss between fp32 and fp64
./compare_unifrac_matrix.py test500.weighted_unnormalized_fp32.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.weighted_unnormalized_fp32.h5 t1.h5 3 0.1
rm -f t1.h5
# weighted_normalized
time ssu -f -m weighted_normalized -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 -o t1.h5
./compare_unifrac_matrix.py test500.weighted_normalized_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.weighted_normalized_fp32.f.h5 t1.h5 3 0.1
rm -f t1.h5
time ssu -f -m weighted_normalized_fp64 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp64 -o t1.h5
./compare_unifrac_matrix.py test500.weighted_normalized_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.weighted_normalized_fp32.f.h5 t1.h5 3 0.1
rm -f t1.h5
# unweighted_unnormalized
time ssu -f -m unweighted_unnormalized -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 -o t1.h5
./compare_unifrac_matrix.py test500.unweighted_unnormalized_fp32.f.h5 t1.h5 1.e-4
rm -f t1.h5
# unweighted
time ssu -f -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 -o t1.h5
./compare_unifrac_matrix.py test500.unweighted_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
rm -f t1.h5
time ssu -f -m unweighted_fp64 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp64 -o t1.h5
./compare_unifrac_matrix.py test500.unweighted_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
# hdf5 without distance matrix, just PCoA
echo "hdf5_nodist"
time ssu -f -m unweighted_fp64 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_nodist -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
time ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_nodist -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
# permanova
echo "permanova"
time ssu -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5 --permanova 999 -g test500.tsv -c empo_2 -o t1.h5
# compare to values given by skbio.stats.distance.permanova
python compare_unifrac_stats.py t1.h5 5 999 1.001112 0.456 0.001 0.1
ls -l t1.h5
rm -f t1.h5
time ssu -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5 --permanova 99 -g test500.tsv -c empo_2 -o t1.h5
python compare_unifrac_stats.py t1.h5 5 99 1.001112 0.456 0.001 0.2
ls -l t1.h5
rm -f t1.h5
time ssu -m weighted_unnormalized_fp32 -i test500.biom -t test500.tre --pcoa 4 -r hdf5_nodist -g test500.tsv -c empo_3 -o t1.h5
# compare to values given by skbio.stats.distance.permanova
python compare_unifrac_stats.py t1.h5 17 999 0.890697 0.865 0.001 0.1
ls -l t1.h5
rm -f t1.h5
# partials
echo "partials"
ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --mode partial-report --n-partials 2
time ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --mode partial --start 0 --stop 125 -o t1.partial.1
time ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --mode partial --start 125 --stop 250 -o t1.partial.2
ls -l t1.partial*
ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --mode check-partial --partial-pattern 't1.partial.*'
time ssu -f -m unweighted_fp64 -i test500.biom -t test500.tre --pcoa 4 --mode merge-partial --partial-pattern 't1.partial.*' -r hdf5_fp64 -o t1.h5
./compare_unifrac_matrix.py test500.unweighted_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
time ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --pcoa 4 --mode merge-partial --partial-pattern 't1.partial.*' -r hdf5_fp32 -o t1.h5
./compare_unifrac_matrix.py test500.unweighted_fp32.f.h5 t1.h5 1.e-5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
time ssu -f -m unweighted_fp32 -i test500.biom -t test500.tre --pcoa 4 --mode merge-partial --partial-pattern 't1.partial.*' -r hdf5_nodist -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.1
ls -l t1.h5
rm -f t1.h5
rm -f t1.partial.*
# subsample
echo "subsample"
time ssu -f -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 --subsample-depth 100 -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.3
rm -f t1.h5
time ssu -f -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 --subsample-depth 100 --subsample-replacement with -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.3
rm -f t1.h5
time ssu -f -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5_fp32 --subsample-depth 100 --subsample-replacement without -o t1.h5
./compare_unifrac_pcoa.py test500.unweighted_fp32.f.h5 t1.h5 3 0.3
rm -f t1.h5
time ssu -m unweighted -i test500.biom -t test500.tre --pcoa 4 -r hdf5_nodist --permanova 99 -g test500.tsv -c empo_2 --subsample-depth 100 -o t1.h5
python compare_unifrac_stats.py t1.h5 5 99 1.001112 0.456 0.05 0.6
ls -l t1.h5
rm -f t1.h5
# multi
echo "multi"
time ssu -f -m unweighted -i test500.biom -t test500.tre --pcoa 4 --mode multi --subsample-depth 100 --n-subsamples 10 -o t1.h5
./compare_unifrac_pcoa_multi.py test500.unweighted_fp32.f.h5 t1.h5 10 3 0.3
ls -l t1.h5
rm -f t1.h5
time ssu -m unweighted -i test500.biom -t test500.tre --pcoa 4 --mode multi --n-subsamples 10 --permanova 99 -g test500.tsv -c empo_2 --subsample-depth 100 -o t1.h5
python compare_unifrac_stats_multi.py t1.h5 10 5 99 1.001112 0.456 0.08 0.6
ls -l t1.h5
rm -f t1.h5
time ssu -m unweighted -i test500.biom -t test500.tre --pcoa 4 --mode multi --n-subsamples 10 --permanova 99 -g test500.tsv -c empo_2 --subsample-depth 100 --subsample-replacement without -o t1.h5
python compare_unifrac_stats_multi.py t1.h5 10 5 99 1.001112 0.456 0.08 0.6
ls -l t1.h5
rm -f t1.h5
popd
# finally, test rapi, too
pushd src
if [[ "$(uname -s)" == "Linux" ]];
then
make clean
rm -f ~/.R/Makevars
conda install -q --yes --strict-channel-priority -c conda-forge r-base
unset CXXFLAGS
unset CFLAGS
unset DEBUG_CXXFLAGS
unset DEBUG_CFLAGS
# the r-base package has a broken dependency
ln -s $CONDA_PREFIX/lib/libreadline.so $CONDA_PREFIX/lib/libreadline.so.6
R -e 'install.packages("Rcpp", repos="http://lib.stat.cmu.edu/R/CRAN/")'
make rapi_test
fi
popd
- name: Sanity checks
shell: bash -l {0}
run: |
conda activate unifrac
# keep it low for runs in containers
export OMP_NUM_THREADS=2
pushd ci
./crawford_test.sh
popd