Skip to content

Commit 11e2c7d

Browse files
dcheriandcherian
committed
Merge branch 'main' into optimize-engine-flox
* main: (64 commits) import `normalize_axis_index` from `numpy.lib` on `numpy>=2` (#364) Optimize `min_count` when `expected_groups` is not provided. (#236) Use threadpool for finding labels in chunk (#327) Manually fuse reindexing intermediates with blockwise reduction for cohorts. (#300) Bump codecov/codecov-action from 4.1.1 to 4.3.1 (#362) Add cubed notebook for hourly climatology example using "map-reduce" method (#356) Optimize bitmask finding for chunk size 1 and single chunk cases (#360) Edits to climatology doc (#361) Fix benchmarks (#358) Trim CI (#355) [pre-commit.ci] pre-commit autoupdate (#350) Initial minimal working Cubed example for "map-reduce" (#352) Bump codecov/codecov-action from 4.1.0 to 4.1.1 (#349) `method` heuristics: Avoid dot product as much as possible (#347) Fix nanlen with strings (#344) Fix direct quantile reduction (#343) Fix upstream-dev CI, silence warnings (#341) Bump codecov/codecov-action from 4.0.0 to 4.1.0 (#338) Fix direct reductions of Xarray objects (#339) Test with py3.12 (#336) ...
2 parents 4ab01d3 + fb180e8 commit 11e2c7d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2302
-773
lines changed

‎.github/workflows/benchmarks.yml

+8-4
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,19 @@ jobs:
4141
OMP_NUM_THREADS: 1
4242
ASV_FACTOR: 1.5
4343
ASV_SKIP_SLOW: 1
44+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
45+
LAST_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
46+
HEAD_LABEL: ${{ github.event.pull_request.head.label }}
47+
BASE_LABEL: ${{ github.event.pull_request.base.label }}
4448
run: |
4549
# set -x
4650
# ID this runner
4751
asv machine --yes
48-
echo "Baseline: ${{ github.event.pull_request.base.sha }} (${{ github.event.pull_request.base.label }})"
49-
echo "Contender: ${GITHUB_SHA} (${{ github.event.pull_request.head.label }})"
52+
echo "Baseline: $LAST_HEAD_SHA ($BASE_LABEL)"
53+
echo "Contender: ${GITHUB_SHA} ($HEAD_LABEL)"
5054
# Run benchmarks for current commit against base
5155
ASV_OPTIONS="--split --show-stderr --factor $ASV_FACTOR"
52-
asv continuous $ASV_OPTIONS ${{ github.event.pull_request.base.sha }} ${GITHUB_SHA} \
56+
asv continuous $ASV_OPTIONS $BASE_SHA ${GITHUB_SHA} \
5357
| sed "/Traceback \|failed$\|PERFORMANCE DECREASED/ s/^/::error::/" \
5458
| tee benchmarks.log
5559
# Report and export results for subsequent steps
@@ -64,7 +68,7 @@ jobs:
6468
cp benchmarks/README_CI.md benchmarks.log .asv/results/
6569
working-directory: ${{ env.ASV_DIR }}
6670

67-
- uses: actions/upload-artifact@v3
71+
- uses: actions/upload-artifact@v4
6872
if: always()
6973
with:
7074
name: asv-benchmark-results-${{ runner.os }}

‎.github/workflows/ci-additional.yaml

+6-6
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141

4242
env:
4343
CONDA_ENV_FILE: ci/environment.yml
44-
PYTHON_VERSION: "3.11"
44+
PYTHON_VERSION: "3.12"
4545

4646
steps:
4747
- uses: actions/checkout@v4
@@ -59,7 +59,7 @@ jobs:
5959
environment-name: flox-tests
6060
init-shell: bash
6161
cache-environment: true
62-
cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
62+
cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
6363
create-args: |
6464
python=${{ env.PYTHON_VERSION }}
6565
@@ -77,7 +77,7 @@ jobs:
7777
--ignore flox/tests \
7878
--cov=./ --cov-report=xml
7979
- name: Upload code coverage to Codecov
80-
uses: codecov/codecov-action@v3.1.4
80+
uses: codecov/codecov-action@v4.3.1
8181
with:
8282
file: ./coverage.xml
8383
flags: unittests
@@ -95,7 +95,7 @@ jobs:
9595
shell: bash -l {0}
9696
env:
9797
CONDA_ENV_FILE: ci/environment.yml
98-
PYTHON_VERSION: "3.11"
98+
PYTHON_VERSION: "3.12"
9999

100100
steps:
101101
- uses: actions/checkout@v4
@@ -112,7 +112,7 @@ jobs:
112112
environment-name: flox-tests
113113
init-shell: bash
114114
cache-environment: true
115-
cache-env-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
115+
cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
116116
create-args: |
117117
python=${{ env.PYTHON_VERSION }}
118118
- name: Install flox
@@ -131,7 +131,7 @@ jobs:
131131
python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
132132
133133
- name: Upload mypy coverage to Codecov
134-
uses: codecov/codecov-action@v3.1.4
134+
uses: codecov/codecov-action@v4.3.1
135135
with:
136136
file: mypy_report/cobertura.xml
137137
flags: mypy

‎.github/workflows/ci.yaml

+20-11
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,11 @@ jobs:
2424
strategy:
2525
fail-fast: false
2626
matrix:
27-
os: ["ubuntu-latest", "windows-latest"]
28-
python-version: ["3.9", "3.11"]
27+
os: ["ubuntu-latest"]
28+
python-version: ["3.9", "3.12"]
29+
include:
30+
- os: "windows-latest"
31+
python-version: "3.12"
2932
steps:
3033
- uses: actions/checkout@v4
3134
with:
@@ -49,7 +52,7 @@ jobs:
4952
run: |
5053
pytest -n auto --cov=./ --cov-report=xml
5154
- name: Upload code coverage to Codecov
52-
uses: codecov/codecov-action@v3.1.4
55+
uses: codecov/codecov-action@v4.3.1
5356
with:
5457
file: ./coverage.xml
5558
flags: unittests
@@ -66,11 +69,9 @@ jobs:
6669
strategy:
6770
fail-fast: false
6871
matrix:
69-
python-version: ["3.11"]
70-
env: ["no-xarray", "no-dask"]
72+
python-version: ["3.12"]
73+
env: ["no-dask"] # "no-xarray", "no-numba"
7174
include:
72-
- env: "no-numba"
73-
python-version: "3.12"
7475
- env: "minimal-requirements"
7576
python-version: "3.9"
7677
steps:
@@ -93,7 +94,7 @@ jobs:
9394
run: |
9495
python -m pytest -n auto --cov=./ --cov-report=xml
9596
- name: Upload code coverage to Codecov
96-
uses: codecov/codecov-action@v3.1.4
97+
uses: codecov/codecov-action@v4.3.1
9798
with:
9899
file: ./coverage.xml
99100
flags: unittests
@@ -119,8 +120,9 @@ jobs:
119120
environment-name: xarray-tests
120121
init-shell: bash
121122
cache-environment: true
122-
create-args: |
123-
python=3.10
123+
create-args: >-
124+
python=3.11
125+
pint>=0.22
124126
- name: Install xarray
125127
run: |
126128
python -m pip install --no-deps .
@@ -144,4 +146,11 @@ jobs:
144146
id: status
145147
run: |
146148
set -euo pipefail
147-
python -m pytest -n auto xarray/tests/test_groupby.py
149+
python -m pytest -n auto \
150+
xarray/tests/test_groupby.py \
151+
xarray/tests/test_units.py::TestDataArray::test_computation_objects \
152+
xarray/tests/test_units.py::TestDataArray::test_grouped_operations \
153+
xarray/tests/test_units.py::TestDataArray::test_resample \
154+
xarray/tests/test_units.py::TestDataset::test_computation_objects \
155+
xarray/tests/test_units.py::TestDataset::test_grouped_operations \
156+
xarray/tests/test_units.py::TestDataset::test_resample

‎.github/workflows/pypi.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ jobs:
1010
steps:
1111
- uses: actions/checkout@v4
1212
- name: Set up Python
13-
uses: actions/setup-python@v4
13+
uses: actions/setup-python@v5
1414
with:
1515
python-version: "3.x"
1616
- name: Install dependencies

‎.github/workflows/testpypi-release.yaml

+6-6
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ jobs:
2121
with:
2222
fetch-depth: 0
2323

24-
- uses: actions/setup-python@v4
24+
- uses: actions/setup-python@v5
2525
name: Install Python
2626
with:
27-
python-version: "3.11"
27+
python-version: "3.12"
2828

2929
- name: Install dependencies
3030
run: |
@@ -53,7 +53,7 @@ jobs:
5353
echo "✅ Looks good"
5454
fi
5555
56-
- uses: actions/upload-artifact@v3
56+
- uses: actions/upload-artifact@v4
5757
with:
5858
name: releases
5959
path: dist
@@ -62,11 +62,11 @@ jobs:
6262
needs: build-artifacts
6363
runs-on: ubuntu-latest
6464
steps:
65-
- uses: actions/setup-python@v4
65+
- uses: actions/setup-python@v5
6666
name: Install Python
6767
with:
68-
python-version: "3.11"
69-
- uses: actions/download-artifact@v3
68+
python-version: "3.12"
69+
- uses: actions/download-artifact@v4
7070
with:
7171
name: releases
7272
path: dist

‎.github/workflows/upstream-dev-ci.yaml

+45-4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ on:
77
types: [opened, reopened, synchronize, labeled]
88
branches:
99
- main
10+
paths:
11+
- ".github/workflows/upstream-dev-ci.yaml"
12+
- "ci/upstream-dev-env.yml"
13+
- "flox/*"
1014
schedule:
1115
- cron: "0 0 * * *" # Daily “At 00:00” UTC
1216
workflow_dispatch: # allows you to trigger the workflow run manually
@@ -19,14 +23,18 @@ jobs:
1923
upstream-dev:
2024
name: upstream-dev
2125
runs-on: ubuntu-latest
22-
if: ${{ (contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request') || github.event_name == 'workflow_dispatch' }}
26+
if: ${{
27+
(contains(github.event.pull_request.labels.*.name, 'test-upstream') && github.event_name == 'pull_request')
28+
|| github.event_name == 'workflow_dispatch'
29+
|| github.event_name == 'schedule'
30+
}}
2331
defaults:
2432
run:
2533
shell: bash -l {0}
2634
strategy:
2735
fail-fast: false
2836
matrix:
29-
python-version: ["3.11"]
37+
python-version: ["3.12"]
3038
steps:
3139
- uses: actions/checkout@v4
3240
with:
@@ -37,16 +45,49 @@ jobs:
3745
- name: Set up conda environment
3846
uses: mamba-org/setup-micromamba@v1
3947
with:
40-
environment-file: ci/upstream-dev-env.yml
4148
environment-name: flox-tests
4249
init-shell: bash
43-
cache-environment: true
50+
# cache-environment: true
51+
# micromamba list does not list pip dependencies, so install mamba
4452
create-args: >-
53+
mamba
54+
pip
4555
python=${{ matrix.python-version }}
4656
pytest-reportlog
57+
58+
- name: Install upstream dev dependencies
59+
run: |
60+
# install cython for building cftime without build isolation
61+
micromamba install -f ci/upstream-dev-env.yml
62+
micromamba remove --force numpy scipy pandas cftime
63+
python -m pip install \
64+
-i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
65+
--no-deps \
66+
--pre \
67+
--upgrade \
68+
numpy \
69+
scipy \
70+
pandas \
71+
xarray
72+
# without build isolation for packages compiling against numpy
73+
# TODO: remove once there are `numpy>=2.0` builds for cftime
74+
python -m pip install \
75+
--no-deps \
76+
--upgrade \
77+
--no-build-isolation \
78+
git+https://github.com/Unidata/cftime
79+
python -m pip install \
80+
git+https://github.com/dask/dask \
81+
git+https://github.com/ml31415/numpy-groupies
82+
4783
- name: Install flox
4884
run: |
4985
python -m pip install --no-deps -e .
86+
87+
- name: List deps
88+
run: |
89+
# micromamba list does not list pip dependencies
90+
mamba list
5091
- name: Run Tests
5192
if: success()
5293
id: status

‎.pre-commit-config.yaml

+25-6
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@ ci:
44
repos:
55
- repo: https://github.com/astral-sh/ruff-pre-commit
66
# Ruff version.
7-
rev: "v0.0.292"
7+
rev: "v0.3.5"
88
hooks:
99
- id: ruff
1010
args: ["--fix", "--show-fixes"]
1111

1212
- repo: https://github.com/pre-commit/mirrors-prettier
13-
rev: "v3.0.3"
13+
rev: "v4.0.0-alpha.8"
1414
hooks:
1515
- id: prettier
1616

@@ -23,7 +23,7 @@ repos:
2323
- id: check-docstring-first
2424

2525
- repo: https://github.com/psf/black-pre-commit-mirror
26-
rev: 23.9.1
26+
rev: 24.3.0
2727
hooks:
2828
- id: black
2929

@@ -36,14 +36,14 @@ repos:
3636
- mdformat-myst
3737

3838
- repo: https://github.com/nbQA-dev/nbQA
39-
rev: 1.7.0
39+
rev: 1.8.5
4040
hooks:
4141
- id: nbqa-black
4242
- id: nbqa-ruff
4343
args: [--fix]
4444

4545
- repo: https://github.com/kynan/nbstripout
46-
rev: 0.6.1
46+
rev: 0.7.1
4747
hooks:
4848
- id: nbstripout
4949
args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
@@ -56,6 +56,25 @@ repos:
5656
- tomli
5757

5858
- repo: https://github.com/abravalheri/validate-pyproject
59-
rev: v0.15
59+
rev: v0.16
6060
hooks:
6161
- id: validate-pyproject
62+
63+
- repo: https://github.com/rhysd/actionlint
64+
rev: v1.6.27
65+
hooks:
66+
- id: actionlint
67+
files: ".github/workflows/"
68+
args:
69+
[
70+
"-ignore",
71+
"SC1090",
72+
"-ignore",
73+
"SC2046",
74+
"-ignore",
75+
"SC2086",
76+
"-ignore",
77+
"SC2129",
78+
"-ignore",
79+
"SC2155",
80+
]

‎asv_bench/asv.conf.json

+5
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@
2727
// "python setup.py build",
2828
// "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
2929
// ],
30+
//
31+
"build_command": [
32+
"python setup.py build",
33+
"python -mpip wheel --no-deps --no-build-isolation --no-index -w {build_cache_dir} {build_dir}"
34+
],
3035

3136
// List of branches to benchmark. If not provided, defaults to "master"
3237
// (for git) or "default" (for mercurial).

0 commit comments

Comments
 (0)