Skip to content

Commit 28a0ad0

Browse files
jpn--dhensle
jpn--
and
dhensle
authored
Changes to support sharrow on 2-zone model (ActivitySim#867)
* test external regional model examples * cache buster * optional variable doc * fix conda cache dirs * trip_destination alts preprocessor * non_hh_veh cat, drop unused cols for alts * blacken * adding missed alts columns used in xborder model * remove unneeded addition to categorical * clearer time logging * bump required numba to 0.57 for np.nan_to_num * sharrow docs * use compute_setting in sharrow debugging * fix comment * debug helper values * dtype compute fixes * land_use_columns_orig * fix and test orig_land_use with explicit chunking * repair * add missing test result file * omx_ignore_patterns * revert change to drop size terms * creating separate sample and simulate preprocessors * bugfix * skim_dataset loading without dask * require sharrow 2.9 * wait to close open files * require sharrow 2.9.1 * landuse index sort before sharrow recode check * decode time periods * use original tazs where possible * update numba in envs to 0.57 * no fastmath in tour mode choice * sharrow cache by version * include sharrow setting in log by defualt * use dask if required * store_skims_in_shm setting --------- Co-authored-by: David Hensle <[email protected]> Co-authored-by: David Hensle <[email protected]>
1 parent bd48d3d commit 28a0ad0

19 files changed

+386
-208
lines changed

‎.github/workflows/core_tests.yml

+46-115
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010
- '*'
1111

1212
env:
13-
CACHE_NUMBER: 0 # increase to reset cache manually
13+
CACHE_NUMBER: 2 # increase to reset cache manually
1414

1515
jobs:
1616
foundation:
@@ -24,10 +24,10 @@ jobs:
2424
name: linux-64-py${{ matrix.python-version }}
2525
runs-on: ubuntu-latest
2626
steps:
27-
- uses: actions/checkout@v3
27+
- uses: actions/checkout@v4
2828

2929
- name: Setup Mambaforge
30-
uses: conda-incubator/setup-miniconda@v2
30+
uses: conda-incubator/setup-miniconda@v3
3131
with:
3232
miniforge-variant: Mambaforge
3333
miniforge-version: latest
@@ -38,7 +38,7 @@ jobs:
3838
- name: Set cache date for year and month
3939
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
4040

41-
- uses: actions/cache@v3
41+
- uses: actions/cache@v4
4242
with:
4343
path: /usr/share/miniconda3/envs/asim-test
4444
key: linux-64-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
@@ -47,19 +47,6 @@ jobs:
4747
- name: Update environment
4848
run: |
4949
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
50-
mamba install --yes \
51-
"psutil=5.9.5" \
52-
"pydantic=2.6.1" \
53-
"pypyr=5.8.0" \
54-
"pytables=3.6.1" \
55-
"pytest-cov" \
56-
"pytest-regressions=2.5.0" \
57-
"scikit-learn=1.2.2" \
58-
"sharrow>=2.6.0" \
59-
"simwrapper=1.8.5" \
60-
"xarray=2023.2.0" \
61-
"zarr=2.14.2" \
62-
"zstandard=0.21.0"
6350
if: steps.cache.outputs.cache-hit != 'true'
6451

6552
- name: Install activitysim
@@ -109,12 +96,10 @@ jobs:
10996
include:
11097
- os: macos-latest
11198
label: macOS
112-
prefix: /Users/runner/miniconda3/envs/asim-test
11399
python-version: "3.10"
114100

115101
- os: windows-latest
116102
label: win-64
117-
prefix: C:\Miniconda3\envs\asim-test
118103
python-version: "3.10"
119104

120105
defaults:
@@ -124,10 +109,10 @@ jobs:
124109
name: ${{ matrix.label }}
125110
runs-on: ${{ matrix.os }}
126111
steps:
127-
- uses: actions/checkout@v3
112+
- uses: actions/checkout@v4
128113

129114
- name: Setup Mambaforge
130-
uses: conda-incubator/setup-miniconda@v2
115+
uses: conda-incubator/setup-miniconda@v3
131116
with:
132117
miniforge-variant: Mambaforge
133118
miniforge-version: latest
@@ -138,28 +123,15 @@ jobs:
138123
- name: Set cache date for year and month
139124
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
140125

141-
- uses: actions/cache@v3
126+
- uses: actions/cache@v4
142127
with:
143-
path: ${{ matrix.prefix }}
128+
path: ${{ env.CONDA }}/envs
144129
key: ${{ matrix.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
145130
id: cache
146131

147132
- name: Update environment
148133
run: |
149134
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
150-
mamba install --yes \
151-
"psutil=5.9.5" \
152-
"pydantic=2.6.1" \
153-
"pypyr=5.8.0" \
154-
"pytables=3.6.1" \
155-
"pytest-cov" \
156-
"pytest-regressions=2.5.0" \
157-
"scikit-learn=1.2.2" \
158-
"sharrow>=2.6.0" \
159-
"simwrapper=1.8.5" \
160-
"xarray=2023.2.0" \
161-
"zarr=2.14.2" \
162-
"zstandard=0.21.0"
163135
if: steps.cache.outputs.cache-hit != 'true'
164136

165137
- name: Install activitysim
@@ -199,7 +171,6 @@ jobs:
199171
builtin_regional_models:
200172
needs: foundation
201173
env:
202-
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
203174
python-version: "3.10"
204175
label: linux-64
205176
strategy:
@@ -221,10 +192,10 @@ jobs:
221192
name: ${{ matrix.region }}
222193
runs-on: ubuntu-latest
223194
steps:
224-
- uses: actions/checkout@v3
195+
- uses: actions/checkout@v4
225196

226197
- name: Setup Mambaforge
227-
uses: conda-incubator/setup-miniconda@v2
198+
uses: conda-incubator/setup-miniconda@v3
228199
with:
229200
miniforge-variant: Mambaforge
230201
miniforge-version: latest
@@ -235,28 +206,15 @@ jobs:
235206
- name: Set cache date for year and month
236207
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
237208

238-
- uses: actions/cache@v3
209+
- uses: actions/cache@v4
239210
with:
240-
path: ${{ env.mamba-env-prefix }}
211+
path: ${{ env.CONDA }}/envs
241212
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
242213
id: cache
243214

244215
- name: Update environment
245216
run: |
246217
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
247-
mamba install --yes \
248-
"psutil=5.9.5" \
249-
"pydantic=2.6.1" \
250-
"pypyr=5.8.0" \
251-
"pytables=3.6.1" \
252-
"pytest-cov" \
253-
"pytest-regressions=2.5.0" \
254-
"scikit-learn=1.2.2" \
255-
"sharrow>=2.6.0" \
256-
"simwrapper=1.8.5" \
257-
"xarray=2023.2.0" \
258-
"zarr=2.14.2" \
259-
"zstandard=0.21.0"
260218
if: steps.cache.outputs.cache-hit != 'true'
261219

262220
- name: Install activitysim
@@ -301,25 +259,31 @@ jobs:
301259
external_regional_models:
302260
needs: foundation
303261
env:
304-
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
305262
python-version: "3.10"
306263
label: linux-64
307264
strategy:
308265
matrix:
309-
region:
310-
- prototype_mtc
311-
- prototype_psrc_in_development
266+
include:
267+
- region: Standard 1-Zone Example (MTC)
268+
region-org: ActivitySim
269+
region-repo: activitysim-prototype-mtc
270+
region-branch: extended
271+
- region: Standard 2-Zone Example (SANDAG)
272+
region-org: ActivitySim
273+
region-repo: sandag-abm3-example
274+
region-branch: main
312275
fail-fast: false
313276
defaults:
314277
run:
315278
shell: bash -l {0}
316279
name: ${{ matrix.region }}
317280
runs-on: ubuntu-latest
318281
steps:
319-
- uses: actions/checkout@v3
282+
- name: Checkout ActivitySim
283+
uses: actions/checkout@v4
320284

321285
- name: Setup Mambaforge
322-
uses: conda-incubator/setup-miniconda@v2
286+
uses: conda-incubator/setup-miniconda@v3
323287
with:
324288
miniforge-variant: Mambaforge
325289
miniforge-version: latest
@@ -330,52 +294,46 @@ jobs:
330294
- name: Set cache date for year and month
331295
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
332296

333-
- uses: actions/cache@v3
297+
- uses: actions/cache@v4
334298
with:
335299
path: |
336-
${{ env.mamba-env-prefix }}
300+
${{ env.CONDA }}/envs
337301
~/.cache/ActivitySim
338302
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
339303
id: cache
340304

341305
- name: Update environment
342306
run: |
343307
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
344-
mamba install --yes \
345-
"psutil=5.9.5" \
346-
"pydantic=2.6.1" \
347-
"pypyr=5.8.0" \
348-
"pytables=3.6.1" \
349-
"pytest-cov" \
350-
"pytest-regressions=2.5.0" \
351-
"scikit-learn=1.2.2" \
352-
"sharrow>=2.6.0" \
353-
"simwrapper=1.8.5" \
354-
"xarray=2023.2.0" \
355-
"zarr=2.14.2" \
356-
"zstandard=0.21.0"
357308
if: steps.cache.outputs.cache-hit != 'true'
358309

359310
- name: Install activitysim
360311
# installing without dependencies is faster, we trust that all needed dependencies
361312
# are in the conda environment defined above. Also, this avoids pip getting
362313
# confused and reinstalling tables (pytables).
363314
run: |
364-
python -m pip install -e . --no-deps
315+
python -m pip install . --no-deps
365316
366317
- name: Conda checkup
367318
run: |
368319
mamba info -a
369320
mamba list
370321
322+
- name: Checkout Example
323+
uses: actions/checkout@v4
324+
with:
325+
repository: '${{ matrix.region-org }}/${{ matrix.region-repo }}'
326+
ref: '${{ matrix.region-branch }}'
327+
path: '${{ matrix.region-repo }}'
328+
371329
- name: Test ${{ matrix.region }}
372330
run: |
373-
python -m activitysim test ${{ matrix.region }}
331+
cd ${{ matrix.region-repo }}/test
332+
python -m pytest .
374333
375334
random_seed_generation:
376335
needs: foundation
377336
env:
378-
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
379337
python-version: "3.10"
380338
label: linux-64
381339
defaults:
@@ -385,10 +343,10 @@ jobs:
385343
runs-on: ubuntu-latest
386344

387345
steps:
388-
- uses: actions/checkout@v3
346+
- uses: actions/checkout@v4
389347

390348
- name: Setup Mambaforge
391-
uses: conda-incubator/setup-miniconda@v2
349+
uses: conda-incubator/setup-miniconda@v3
392350
with:
393351
miniforge-variant: Mambaforge
394352
miniforge-version: latest
@@ -399,28 +357,15 @@ jobs:
399357
- name: Set cache date for year and month
400358
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
401359

402-
- uses: actions/cache@v3
360+
- uses: actions/cache@v4
403361
with:
404-
path: ${{ env.mamba-env-prefix }}
362+
path: ${{ env.CONDA }}/envs
405363
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
406364
id: cache
407365

408366
- name: Update environment
409367
run: |
410368
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
411-
mamba install --yes \
412-
"psutil=5.9.5" \
413-
"pydantic=2.6.1" \
414-
"pypyr=5.8.0" \
415-
"pytables=3.6.1" \
416-
"pytest-cov" \
417-
"pytest-regressions=2.5.0" \
418-
"scikit-learn=1.2.2" \
419-
"sharrow>=2.6.0" \
420-
"simwrapper=1.8.5" \
421-
"xarray=2023.2.0" \
422-
"zarr=2.14.2" \
423-
"zstandard=0.21.0"
424369
if: steps.cache.outputs.cache-hit != 'true'
425370

426371
- name: Install activitysim
@@ -442,7 +387,6 @@ jobs:
442387
estimation_mode:
443388
needs: foundation
444389
env:
445-
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
446390
python-version: "3.10"
447391
label: linux-64
448392
defaults:
@@ -451,10 +395,10 @@ jobs:
451395
name: estimation_mode_test
452396
runs-on: ubuntu-latest
453397
steps:
454-
- uses: actions/checkout@v3
398+
- uses: actions/checkout@v4
455399

456400
- name: Setup Mambaforge
457-
uses: conda-incubator/setup-miniconda@v2
401+
uses: conda-incubator/setup-miniconda@v3
458402
with:
459403
miniforge-variant: Mambaforge
460404
miniforge-version: latest
@@ -465,28 +409,15 @@ jobs:
465409
- name: Set cache date for year and month
466410
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
467411

468-
- uses: actions/cache@v3
412+
- uses: actions/cache@v4
469413
with:
470-
path: ${{ env.mamba-env-prefix }}
414+
path: ${{ env.CONDA }}/envs
471415
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
472416
id: cache
473417

474418
- name: Update environment
475419
run: |
476420
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
477-
mamba install --yes \
478-
"psutil=5.9.5" \
479-
"pydantic=2.6.1" \
480-
"pypyr=5.8.0" \
481-
"pytables=3.6.1" \
482-
"pytest-cov" \
483-
"pytest-regressions=2.5.0" \
484-
"scikit-learn=1.2.2" \
485-
"sharrow>=2.6.0" \
486-
"simwrapper=1.8.5" \
487-
"xarray=2023.2.0" \
488-
"zarr=2.14.2" \
489-
"zstandard=0.21.0"
490421
if: steps.cache.outputs.cache-hit != 'true'
491422

492423
- name: Install Larch
@@ -517,15 +448,15 @@ jobs:
517448
run:
518449
shell: bash -l {0}
519450
steps:
520-
- uses: actions/checkout@v3
451+
- uses: actions/checkout@v4
521452
with:
522453
fetch-depth: 0 # get all tags, lets setuptools_scm do its thing
523454
- name: Set up Python 3.10
524455
uses: actions/setup-python@v2
525456
with:
526457
python-version: "3.10"
527458
- name: Install dependencies
528-
uses: conda-incubator/setup-miniconda@v2
459+
uses: conda-incubator/setup-miniconda@v3
529460
with:
530461
miniforge-variant: Mambaforge
531462
miniforge-version: latest

0 commit comments

Comments
 (0)