Merge branch 'main' into flox-quantile

* main: (42 commits)
  correctly encode/decode _FillValues/missing_values/dtypes for packed data (pydata#8713)
  Expand use of `.oindex` and `.vindex` (pydata#8790)
  Return a dataclass from Grouper.factorize (pydata#8777)
  [skip-ci] Fix upstream-dev env (pydata#8839)
  Add dask-expr for windows envs (pydata#8837)
  [skip-ci] Add dask-expr dependency to doc.yml (pydata#8835)
  Add `dask-expr` to environment-3.12.yml (pydata#8827)
  Make list_chunkmanagers more resilient to broken entrypoints (pydata#8736)
  Do not attempt to broadcast when global option ``arithmetic_broadcast=False`` (pydata#8784)
  try to get the `upstream-dev` CI to complete again (pydata#8823)
  Bump the actions group with 1 update (pydata#8818)
  Update documentation for clarity (pydata#8817)
  DOC: link to zarr.convenience.consolidate_metadata (pydata#8816)
  Refactor Grouper objects (pydata#8776)
  Grouper object design doc (pydata#8510)
  Bump the actions group with 2 updates (pydata#8804)
  tokenize() should ignore difference between None and {} attrs (pydata#8797)
  fix: remove Coordinate from __all__ in xarray/__init__.py (pydata#8791)
  Fix non-nanosecond casting behavior for `expand_dims` (pydata#8782)
  Migrate treenode module. (pydata#8757)
  ...

Author: dcherian

.github/workflows/ci-additional.yaml

@@ -128,7 +128,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy
@@ -182,7 +182,7 @@ jobs:
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report xarray/
 
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy39
@@ -243,7 +243,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: pyright_report/cobertura.xml
           flags: pyright
@@ -302,7 +302,7 @@ jobs:
           python -m pyright xarray/
 
       - name: Upload pyright coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: pyright_report/cobertura.xml
           flags: pyright39

.github/workflows/ci.yaml

@@ -143,7 +143,7 @@ jobs:
           path: pytest.xml
 
       - name: Upload code coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: ./coverage.xml
           flags: unittests

.github/workflows/nightly-wheels.yml

@@ -38,7 +38,7 @@ jobs:
           fi
 
       - name: Upload wheel
-        uses: scientific-python/upload-nightly-action@6e9304f7a3a5501c6f98351537493ec898728299 # 0.3.0
+        uses: scientific-python/upload-nightly-action@b67d7fcc0396e1128a474d1ab2b48aa94680f9fc # 0.5.0
         with:
           anaconda_nightly_upload_token: ${{ secrets.ANACONDA_NIGHTLY }}
           artifacts_path: dist

.github/workflows/pypi-release.yaml

@@ -88,7 +88,7 @@ jobs:
           path: dist
       - name: Publish package to TestPyPI
         if: github.event_name == 'push'
-        uses: pypa/[email protected].11
+        uses: pypa/[email protected].14
         with:
           repository_url: https://test.pypi.org/legacy/
           verbose: true
@@ -111,6 +111,6 @@ jobs:
           name: releases
           path: dist
       - name: Publish package to PyPI
-        uses: pypa/[email protected].11
+        uses: pypa/[email protected].14
         with:
           verbose: true

.github/workflows/upstream-dev-ci.yaml

@@ -52,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -143,7 +143,7 @@ jobs:
         run: |
           python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report
       - name: Upload mypy coverage to Codecov
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v4.1.0
         with:
           file: mypy_report/cobertura.xml
           flags: mypy

ci/install-upstream-wheels.sh

@@ -3,11 +3,13 @@
 # install cython for building cftime without build isolation
 micromamba install "cython>=0.29.20" py-cpuinfo
 # temporarily (?) remove numbagg and numba
-micromamba remove -y numba numbagg
+micromamba remove -y numba numbagg sparse
+# temporarily remove numexpr
+micromamba remove -y numexpr
 # temporarily remove backends
-micromamba remove -y cf_units h5py hdf5 netcdf4
+micromamba remove -y cf_units hdf5 h5py netcdf4
 # forcibly remove packages to avoid artifacts
-conda uninstall -y --force \
+micromamba remove -y --force \
     numpy \
     scipy \
     pandas \
@@ -30,8 +32,17 @@ python -m pip install \
     scipy \
     matplotlib \
     pandas
+# for some reason pandas depends on pyarrow already.
+# Remove once a `pyarrow` version compiled with `numpy>=2.0` is on `conda-forge`
+python -m pip install \
+    -i https://pypi.fury.io/arrow-nightlies/ \
+    --prefer-binary \
+    --no-deps \
+    --pre \
+    --upgrade \
+    pyarrow
 # without build isolation for packages compiling against numpy
-# TODO: remove once there are `numpy>=2.0` builds for numcodecs and cftime
+# TODO: remove once there are `numpy>=2.0` builds for these
 python -m pip install \
     --no-deps \
     --upgrade \
@@ -51,13 +62,14 @@ python -m pip install \
     --no-deps \
     --upgrade \
     git+https://github.com/dask/dask \
+    git+https://github.com/dask/dask-expr \
     git+https://github.com/dask/distributed \
     git+https://github.com/zarr-developers/zarr \
     git+https://github.com/pypa/packaging \
     git+https://github.com/hgrecco/pint \
-    git+https://github.com/pydata/sparse \
     git+https://github.com/intake/filesystem_spec \
     git+https://github.com/SciTools/nc-time-axis \
     git+https://github.com/xarray-contrib/flox \
     git+https://github.com/dgasmith/opt_einsum
+    # git+https://github.com/pydata/sparse
     # git+https://github.com/h5netcdf/h5netcdf

ci/requirements/all-but-dask.yml

@@ -5,6 +5,7 @@ channels:
 dependencies:
   - black
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy

ci/requirements/doc.yml

@@ -9,6 +9,7 @@ dependencies:
   - cartopy
   - cfgrib
   - dask-core>=2022.1
+  - dask-expr
   - hypothesis>=6.75.8
   - h5netcdf>=0.13
   - ipykernel

ci/requirements/environment-3.12.yml

@@ -4,20 +4,22 @@ channels:
   - nodefaults
 dependencies:
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy
   - cftime
   - dask-core
+  - dask-expr
   - distributed
   - flox
-  - fsspec!=2021.7.0
+  - fsspec
   - h5netcdf
   - h5py
   - hdf5
   - hypothesis
   - iris
-  - lxml  # Optional dep of pydap
+  - lxml # Optional dep of pydap
   - matplotlib-base
   - nc-time-axis
   - netcdf4

ci/requirements/environment-windows-3.12.yml

@@ -2,20 +2,22 @@ name: xarray-tests
 channels:
   - conda-forge
 dependencies:
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy
   - cftime
   - dask-core
+  - dask-expr
   - distributed
   - flox
-  - fsspec!=2021.7.0
+  - fsspec
   - h5netcdf
   - h5py
   - hdf5
   - hypothesis
   - iris
-  - lxml  # Optional dep of pydap
+  - lxml # Optional dep of pydap
   - matplotlib-base
   - nc-time-axis
   - netcdf4

ci/requirements/environment-windows.yml

@@ -2,20 +2,22 @@ name: xarray-tests
 channels:
   - conda-forge
 dependencies:
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy
   - cftime
   - dask-core
+  - dask-expr
   - distributed
   - flox
-  - fsspec!=2021.7.0
+  - fsspec
   - h5netcdf
   - h5py
   - hdf5
   - hypothesis
   - iris
-  - lxml  # Optional dep of pydap
+  - lxml # Optional dep of pydap
   - matplotlib-base
   - nc-time-axis
   - netcdf4

ci/requirements/environment.yml

@@ -4,14 +4,16 @@ channels:
   - nodefaults
 dependencies:
   - aiobotocore
+  - array-api-strict
   - boto3
   - bottleneck
   - cartopy
   - cftime
   - dask-core
+  - dask-expr # dask raises a deprecation warning without this, breaking doctests
   - distributed
   - flox
-  - fsspec!=2021.7.0
+  - fsspec
   - h5netcdf
   - h5py
   - hdf5
@@ -32,7 +34,7 @@ dependencies:
   - pip
   - pooch
   - pre-commit
-  - pyarrow # pandas makes a deprecation warning without this, breaking doctests
+  - pyarrow # pandas raises a deprecation warning without this, breaking doctests
   - pydap
   - pytest
   - pytest-cov

ci/requirements/min-all-deps.yml

@@ -8,6 +8,7 @@ dependencies:
   # When upgrading python, numpy, or pandas, must also change
   # doc/user-guide/installing.rst, doc/user-guide/plotting.rst and setup.py.
   - python=3.9
+  - array-api-strict=1.0  # dependency for testing the array api compat
   - boto3=1.24
   - bottleneck=1.3
   - cartopy=0.21

conftest.py

@@ -39,3 +39,11 @@ def add_standard_imports(doctest_namespace, tmpdir):
 
     # always switch to the temporary directory, so files get written there
     tmpdir.chdir()
+
+    # Avoid the dask deprecation warning, can remove if CI passes without this.
+    try:
+        import dask
+    except ImportError:
+        pass
+    else:
+        dask.config.set({"dataframe.query-planning": True})