From ee0a57c3054ff1327816d704ae937cd02b3f08f7 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 27 Aug 2024 17:17:22 -0600 Subject: [PATCH 01/19] bump beam --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4108dff..d1ed3668 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam", + "apache-beam >= 2.58.1", "cftime", "dask >= 2021.11.2", "fastparquet", From 5f3337c4fa038506837c5d917d99d43d7143fd10 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Tue, 27 Aug 2024 17:26:39 -0600 Subject: [PATCH 02/19] remove auto numprocess --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bb778a1f..a5707aa7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,7 +49,7 @@ jobs: - name: 🏄‍♂️ Run Tests shell: bash -l {0} run: | - py.test tests -v -n auto \ + py.test tests -v \ --cov=pangeo_forge_recipes --cov-config .coveragerc \ --cov-report term-missing \ --cov-report xml \ From 682c15fc3b65b8cb496029445063a836f0ba85c8 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:11:38 -0600 Subject: [PATCH 03/19] revert --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a5707aa7..bb778a1f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,7 +49,7 @@ jobs: - name: 🏄‍♂️ Run Tests shell: bash -l {0} run: | - py.test tests -v \ + py.test tests -v -n auto \ --cov=pangeo_forge_recipes --cov-config .coveragerc \ --cov-report term-missing \ --cov-report xml \ From 82e40d0f021c156d666568441ae6376a4f9078b7 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:32:25 -0600 Subject: [PATCH 04/19] beam version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d1ed3668..71a386b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam >= 2.58.1", + "apache-beam == 2.58.0", "cftime", "dask >= 2021.11.2", "fastparquet", From 7cc2fcb2b95e37f6fdf09297b9bbf8491e0593fe Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:35:32 -0600 Subject: [PATCH 05/19] 2.57.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 71a386b7..f3727fb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam == 2.58.0", + "apache-beam == 2.57.0", "cftime", "dask >= 2021.11.2", "fastparquet", From f8a336506b15317f33132d555a2e09f1016056e5 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:38:22 -0600 Subject: [PATCH 06/19] fsspec --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f3727fb7..2a644b3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "cftime", "dask >= 2021.11.2", "fastparquet", - "fsspec[http] >= 2023.4.0", + "fsspec[http] >= 2024.6.1", "h5netcdf", "h5py >= 3.3.0", "kerchunk >= 0.0.7", From ec5dd2526033446d0520ec290c9b3039937667ac Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:39:59 -0600 Subject: [PATCH 07/19] h5py --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2a644b3a..b0ada2e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,13 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam == 2.57.0", + "apache-beam >= 2.58.0", "cftime", "dask >= 2021.11.2", "fastparquet", "fsspec[http] >= 2024.6.1", "h5netcdf", - "h5py >= 3.3.0", + "h5py >= 3.11.0", "kerchunk >= 0.0.7", "netcdf4", "numcodecs >= 0.9.0", From 9578f2e8db57a4b6b11fc6dedc8c31bdb7285334 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:42:45 -0600 Subject: [PATCH 08/19] shrug --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b0ada2e6..cae6a141 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,18 +25,18 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam >= 2.58.0", + "apache-beam", "cftime", - "dask >= 2021.11.2", + "dask, "fastparquet", - "fsspec[http] >= 2024.6.1", + "fsspec[http]", "h5netcdf", - "h5py >= 3.11.0", - "kerchunk >= 0.0.7", + "h5py", + "kerchunk", "netcdf4", - "numcodecs >= 0.9.0", - "xarray >= 0.18.0", - "zarr >= 2.12.0", + "numcodecs", + "xarray", + "zarr", ] [project.optional-dependencies] From d97dcb63b0e081f99fb2ad232ca25a44b3c762b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 19:44:15 +0000 Subject: [PATCH 09/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../feedstock/gpcp_from_gcs_dynamic_chunks.py | 6 +++++- examples/feedstock/terraclimate.py | 7 ++++++- pangeo_forge_recipes/patterns.py | 13 +++++++++++- pangeo_forge_recipes/transforms.py | 21 ++++++++++++++++--- tests/test_rechunking.py | 8 ++++++- tests/test_writers.py | 8 ++++++- 6 files changed, 55 insertions(+), 8 deletions(-) diff --git a/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py b/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py index bc9c3fd0..be8349cc 100644 --- a/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py +++ b/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py @@ -6,7 +6,11 @@ import zarr from pangeo_forge_recipes.patterns import ConcatDim, FilePattern -from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray, StoreToZarr +from pangeo_forge_recipes.transforms import ( + OpenURLWithFSSpec, + OpenWithXarray, + StoreToZarr, +) dates = [ d.to_pydatetime().strftime("%Y%m%d") diff --git a/examples/feedstock/terraclimate.py b/examples/feedstock/terraclimate.py index e70d87ea..cc3ed545 100644 --- a/examples/feedstock/terraclimate.py +++ b/examples/feedstock/terraclimate.py @@ -16,7 +16,12 @@ import zarr from pangeo_forge_recipes.patterns import ConcatDim, FilePattern, MergeDim -from pangeo_forge_recipes.transforms import Indexed, OpenURLWithFSSpec, OpenWithXarray, StoreToZarr +from pangeo_forge_recipes.transforms import ( + Indexed, + OpenURLWithFSSpec, + OpenWithXarray, + StoreToZarr, +) # for the example, we only select two years to keep the example small; # this time range can be extended if you are running the recipe yourself. diff --git a/pangeo_forge_recipes/patterns.py b/pangeo_forge_recipes/patterns.py index 9c74c9d8..795a0d29 100644 --- a/pangeo_forge_recipes/patterns.py +++ b/pangeo_forge_recipes/patterns.py @@ -12,7 +12,18 @@ from enum import Enum, auto from hashlib import sha256 from itertools import product -from typing import Any, Callable, ClassVar, Dict, Iterator, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + Union, +) from .serialization import dict_drop_empty, dict_to_sha256 from .types import CombineOp, Dimension, Index, IndexedPosition, Position diff --git a/pangeo_forge_recipes/transforms.py b/pangeo_forge_recipes/transforms.py index 47997886..3fba3585 100644 --- a/pangeo_forge_recipes/transforms.py +++ b/pangeo_forge_recipes/transforms.py @@ -20,11 +20,26 @@ import zarr from kerchunk.combine import MultiZarrToZarr -from .aggregation import XarraySchema, dataset_to_schema, schema_to_template_ds, schema_to_zarr +from .aggregation import ( + XarraySchema, + dataset_to_schema, + schema_to_template_ds, + schema_to_zarr, +) from .combiners import CombineXarraySchemas, MinMaxCountCombineFn from .openers import open_url, open_with_kerchunk, open_with_xarray -from .patterns import CombineOp, Dimension, FileType, Index, augment_index_with_start_stop -from .rechunking import combine_fragments, consolidate_dimension_coordinates, split_fragment +from .patterns import ( + CombineOp, + Dimension, + FileType, + Index, + augment_index_with_start_stop, +) +from .rechunking import ( + combine_fragments, + consolidate_dimension_coordinates, + split_fragment, +) from .storage import CacheFSSpecTarget, FSSpecTarget from .types import Indexed from .writers import ( diff --git a/tests/test_rechunking.py b/tests/test_rechunking.py index c1719ac5..1decf644 100644 --- a/tests/test_rechunking.py +++ b/tests/test_rechunking.py @@ -15,7 +15,13 @@ consolidate_dimension_coordinates, split_fragment, ) -from pangeo_forge_recipes.types import CombineOp, Dimension, Index, IndexedPosition, Position +from pangeo_forge_recipes.types import ( + CombineOp, + Dimension, + Index, + IndexedPosition, + Position, +) from .conftest import split_up_files_by_variable_and_day from .data_generation import make_ds diff --git a/tests/test_writers.py b/tests/test_writers.py index 20dd1047..647a8f33 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -14,7 +14,13 @@ StoreToZarr, WriteCombinedReference, ) -from pangeo_forge_recipes.types import CombineOp, Dimension, Index, IndexedPosition, Position +from pangeo_forge_recipes.types import ( + CombineOp, + Dimension, + Index, + IndexedPosition, + Position, +) from pangeo_forge_recipes.writers import store_dataset_fragment from .data_generation import make_ds From 498d06a8c11fbcc59fae785f68ee4f8113eb6e3f Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 28 Aug 2024 13:45:03 -0600 Subject: [PATCH 10/19] :rocket: --- .github/workflows/main.yaml | 27 ++++++++++++--------------- pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bb778a1f..8920447a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,28 +32,25 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - cache-dependency-path: pyproject.toml - - name: 🎯 Check cache hit - run: echo '${{ steps.setup-python.outputs.cache-hit }}' + # cache-dependency-path: pyproject.toml + # - name: 🎯 Check cache hit + # run: echo '${{ steps.setup-python.outputs.cache-hit }}' - name: 🌈 Install pangeo-forge-recipes package shell: bash -l {0} run: | python -m pip install -e ".[test]" - - name: 🧑‍💻 On the nightly run, test upstream dev versions - if: | - github.event_name == 'schedule' - shell: bash -l {0} - run: | - python -m pip install -Ur ci/requirements-upstream-dev.txt - python -m pip install -U --pre apache-beam + # - name: 🧑‍💻 On the nightly run, test upstream dev versions + # if: | + # github.event_name == 'schedule' + # shell: bash -l {0} + # run: | + # python -m pip install -Ur ci/requirements-upstream-dev.txt + # python -m pip install -U --pre apache-beam - name: 🏄‍♂️ Run Tests shell: bash -l {0} run: | - py.test tests -v -n auto \ - --cov=pangeo_forge_recipes --cov-config .coveragerc \ - --cov-report term-missing \ - --cov-report xml \ - --durations=10 --durations-min=1.0 + py.test tests -v + - name: 🚦 Run Codecov if: | github.event_name == 'push' || diff --git a/pyproject.toml b/pyproject.toml index cae6a141..ce3e4147 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ keywords = ["pangeo", "data"] dependencies = [ "apache-beam", "cftime", - "dask, + "dask", "fastparquet", "fsspec[http]", "h5netcdf", From 3ebdd0c2f89722e534a59734d7293518d7a3f874 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 19:45:27 +0000 Subject: [PATCH 11/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../feedstock/gpcp_from_gcs_dynamic_chunks.py | 6 +----- examples/feedstock/terraclimate.py | 7 +------ pangeo_forge_recipes/patterns.py | 13 +----------- pangeo_forge_recipes/transforms.py | 21 +++---------------- tests/test_rechunking.py | 8 +------ tests/test_writers.py | 8 +------ 6 files changed, 8 insertions(+), 55 deletions(-) diff --git a/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py b/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py index be8349cc..bc9c3fd0 100644 --- a/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py +++ b/examples/feedstock/gpcp_from_gcs_dynamic_chunks.py @@ -6,11 +6,7 @@ import zarr from pangeo_forge_recipes.patterns import ConcatDim, FilePattern -from pangeo_forge_recipes.transforms import ( - OpenURLWithFSSpec, - OpenWithXarray, - StoreToZarr, -) +from pangeo_forge_recipes.transforms import OpenURLWithFSSpec, OpenWithXarray, StoreToZarr dates = [ d.to_pydatetime().strftime("%Y%m%d") diff --git a/examples/feedstock/terraclimate.py b/examples/feedstock/terraclimate.py index cc3ed545..e70d87ea 100644 --- a/examples/feedstock/terraclimate.py +++ b/examples/feedstock/terraclimate.py @@ -16,12 +16,7 @@ import zarr from pangeo_forge_recipes.patterns import ConcatDim, FilePattern, MergeDim -from pangeo_forge_recipes.transforms import ( - Indexed, - OpenURLWithFSSpec, - OpenWithXarray, - StoreToZarr, -) +from pangeo_forge_recipes.transforms import Indexed, OpenURLWithFSSpec, OpenWithXarray, StoreToZarr # for the example, we only select two years to keep the example small; # this time range can be extended if you are running the recipe yourself. diff --git a/pangeo_forge_recipes/patterns.py b/pangeo_forge_recipes/patterns.py index 795a0d29..9c74c9d8 100644 --- a/pangeo_forge_recipes/patterns.py +++ b/pangeo_forge_recipes/patterns.py @@ -12,18 +12,7 @@ from enum import Enum, auto from hashlib import sha256 from itertools import product -from typing import ( - Any, - Callable, - ClassVar, - Dict, - Iterator, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import Any, Callable, ClassVar, Dict, Iterator, List, Optional, Sequence, Tuple, Union from .serialization import dict_drop_empty, dict_to_sha256 from .types import CombineOp, Dimension, Index, IndexedPosition, Position diff --git a/pangeo_forge_recipes/transforms.py b/pangeo_forge_recipes/transforms.py index 3fba3585..47997886 100644 --- a/pangeo_forge_recipes/transforms.py +++ b/pangeo_forge_recipes/transforms.py @@ -20,26 +20,11 @@ import zarr from kerchunk.combine import MultiZarrToZarr -from .aggregation import ( - XarraySchema, - dataset_to_schema, - schema_to_template_ds, - schema_to_zarr, -) +from .aggregation import XarraySchema, dataset_to_schema, schema_to_template_ds, schema_to_zarr from .combiners import CombineXarraySchemas, MinMaxCountCombineFn from .openers import open_url, open_with_kerchunk, open_with_xarray -from .patterns import ( - CombineOp, - Dimension, - FileType, - Index, - augment_index_with_start_stop, -) -from .rechunking import ( - combine_fragments, - consolidate_dimension_coordinates, - split_fragment, -) +from .patterns import CombineOp, Dimension, FileType, Index, augment_index_with_start_stop +from .rechunking import combine_fragments, consolidate_dimension_coordinates, split_fragment from .storage import CacheFSSpecTarget, FSSpecTarget from .types import Indexed from .writers import ( diff --git a/tests/test_rechunking.py b/tests/test_rechunking.py index 1decf644..c1719ac5 100644 --- a/tests/test_rechunking.py +++ b/tests/test_rechunking.py @@ -15,13 +15,7 @@ consolidate_dimension_coordinates, split_fragment, ) -from pangeo_forge_recipes.types import ( - CombineOp, - Dimension, - Index, - IndexedPosition, - Position, -) +from pangeo_forge_recipes.types import CombineOp, Dimension, Index, IndexedPosition, Position from .conftest import split_up_files_by_variable_and_day from .data_generation import make_ds diff --git a/tests/test_writers.py b/tests/test_writers.py index 647a8f33..20dd1047 100644 --- a/tests/test_writers.py +++ b/tests/test_writers.py @@ -14,13 +14,7 @@ StoreToZarr, WriteCombinedReference, ) -from pangeo_forge_recipes.types import ( - CombineOp, - Dimension, - Index, - IndexedPosition, - Position, -) +from pangeo_forge_recipes.types import CombineOp, Dimension, Index, IndexedPosition, Position from pangeo_forge_recipes.writers import store_dataset_fragment from .data_generation import make_ds From 050429854c8c71a085943d68418b742c376084d1 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 12:32:08 -0600 Subject: [PATCH 12/19] Update .github/workflows/main.yaml Co-authored-by: Justus Magin --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8920447a..79803123 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -38,7 +38,7 @@ jobs: - name: 🌈 Install pangeo-forge-recipes package shell: bash -l {0} run: | - python -m pip install -e ".[test]" + python -m pip install -e ".[test]" "apache-beam>=2.54" # - name: 🧑‍💻 On the nightly run, test upstream dev versions # if: | # github.event_name == 'schedule' From 7519f05afd5401a9e536edf531246d2446002fe3 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 12:33:16 -0600 Subject: [PATCH 13/19] pinned beam, removed 3.9 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 79803123..0a4c700a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11"] # add 3.12 steps: - uses: actions/checkout@v4 with: From 144bafcf8689d73628873d9ff4dbc359533ef7f9 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 12:42:48 -0600 Subject: [PATCH 14/19] pin to older kerchunk version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce3e4147..c7d392a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "fsspec[http]", "h5netcdf", "h5py", - "kerchunk", + "kerchunk==0.2.4", "netcdf4", "numcodecs", "xarray", From 56c68a6acd50124c24250f7f34070ceb7296e21d Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 13:05:10 -0600 Subject: [PATCH 15/19] pin beam --- .github/workflows/main.yaml | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0a4c700a..1bfcd377 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11"] # add 3.12 + python-version: ["3.10", "3.11","3.12"] steps: - uses: actions/checkout@v4 with: @@ -38,7 +38,7 @@ jobs: - name: 🌈 Install pangeo-forge-recipes package shell: bash -l {0} run: | - python -m pip install -e ".[test]" "apache-beam>=2.54" + python -m pip install -e ".[test]" # - name: 🧑‍💻 On the nightly run, test upstream dev versions # if: | # github.event_name == 'schedule' diff --git a/pyproject.toml b/pyproject.toml index c7d392a3..17e20165 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam", + "apache-beam>=2.54", "cftime", "dask", "fastparquet", From 7d1cecf42de82d9ea87475d060061e8439908745 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 13:09:12 -0600 Subject: [PATCH 16/19] Update pyproject.toml Co-authored-by: Justus Magin --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 17e20165..4d755c5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "fsspec[http]", "h5netcdf", "h5py", - "kerchunk==0.2.4", + "kerchunk!=0.2.6", "netcdf4", "numcodecs", "xarray", From a11b85c4e64478f7ea08ac9e1007445458420a7c Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 13:29:50 -0600 Subject: [PATCH 17/19] shift beam version to 2.48 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 17e20165..5c3c74ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ license = { text = "Apache-2.0" } keywords = ["pangeo", "data"] dependencies = [ - "apache-beam>=2.54", + "apache-beam>=2.48", "cftime", "dask", "fastparquet", From f3480ad2060c80479ec1e7ede1a20e12609091d4 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 14:33:06 -0600 Subject: [PATCH 18/19] revert main.yaml changes --- .github/workflows/main.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1bfcd377..e45707d9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,25 +32,29 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip - # cache-dependency-path: pyproject.toml - # - name: 🎯 Check cache hit - # run: echo '${{ steps.setup-python.outputs.cache-hit }}' + cache-dependency-path: pyproject.toml + - name: 🎯 Check cache hit + run: echo '${{ steps.setup-python.outputs.cache-hit }}' - name: 🌈 Install pangeo-forge-recipes package shell: bash -l {0} run: | python -m pip install -e ".[test]" - # - name: 🧑‍💻 On the nightly run, test upstream dev versions - # if: | - # github.event_name == 'schedule' - # shell: bash -l {0} - # run: | - # python -m pip install -Ur ci/requirements-upstream-dev.txt - # python -m pip install -U --pre apache-beam + - name: 🧑‍💻 On the nightly run, test upstream dev versions + if: | + github.event_name == 'schedule' + shell: bash -l {0} + run: | + python -m pip install -Ur ci/requirements-upstream-dev.txt + python -m pip install -U --pre apache-beam - name: 🏄‍♂️ Run Tests shell: bash -l {0} run: | - py.test tests -v + py.test tests -v -n auto \ + --cov=pangeo_forge_recipes --cov-config .coveragerc \ + --cov-report term-missing \ + --cov-report xml \ + --durations=10 --durations-min=1.0 - name: 🚦 Run Codecov if: | github.event_name == 'push' || From 59393f6f95083a9d1b7d19d35d7093d6758cf570 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Mon, 7 Oct 2024 14:34:47 -0600 Subject: [PATCH 19/19] lint --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e45707d9..930ccfa6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,8 +49,7 @@ jobs: - name: 🏄‍♂️ Run Tests shell: bash -l {0} run: | - - py.test tests -v -n auto \ + py.test tests -v -n auto \ --cov=pangeo_forge_recipes --cov-config .coveragerc \ --cov-report term-missing \ --cov-report xml \