speed up tests (#584)

- pytest-xdist

- mpl-base

.github/workflows/climpred_testing.yml

@@ -34,9 +34,10 @@ jobs:
         run: conda info
       - name: Conda list
         run: conda list
+      - name: Cache datasets  # pytest-xdist fails otherwise
+        run: python -c "import climpred; climpred.tutorial._cache_all()"
       - name: Run tests
-        run: |
-          pytest --cov=climpred --cov-report=xml --verbose
+        run: pytest -n 4 --durations=20 --cov=climpred --cov-report=xml
       - name: Upload coverage to codecov
         uses: codecov/[email protected]
         with:

ci/requirements/climpred-dev.yml

@@ -1,6 +1,7 @@
 name: climpred-dev
 channels:
   - conda-forge
+  - nodefaults
 dependencies:
   - python>=3.7
   # Documentation
@@ -52,7 +53,7 @@ dependencies:
   - esmtools>=1.1.3
   - xskillscore>=0.0.18
   # Visualization
-  - matplotlib
+  - matplotlib-base
   - nc-time-axis
   - pip
   - pip:

ci/requirements/docs.yml

@@ -1,26 +1,22 @@
 name: climpred-docs
 channels:
   - conda-forge
+  - nodefaults
 dependencies:
   - python=3.8
-  - bottleneck
-  - cartopy>=0.18.0
   - importlib_metadata
-  - jupyterlab
-  - matplotlib
+  - matplotlib-base
   - nbsphinx
   - nc-time-axis
   - netcdf4
-  - numpy
-  - pandas
   - pygments==2.6.1
   - sphinx
   - sphinxcontrib-napoleon
   - sphinx_rtd_theme
   - sphinx-copybutton
   - toolz
-  - tqdm
   - xarray>=0.16.1
+  - xskillscore>=0.0.18
   - pip
   - pip:
     # Install latest version of climpred.

ci/requirements/docs_notebooks.yml

@@ -1,14 +1,15 @@
 name: climpred-docs-notebooks
 channels:
   - conda-forge
+  - nodefaults
 dependencies:
   - python>=3.7
   - xesmf
   - esmpy
   - bottleneck
   - importlib_metadata
   - jupyterlab
-  - matplotlib
+  - matplotlib-base
   - nbsphinx
   - nbstripout
   - nc-time-axis

ci/requirements/minimum-tests.yml

@@ -2,6 +2,7 @@ name: climpred-minimum-tests
 channels:
   - conda-forge
   - numba/label/dev # Temporary fix to get Numba on Python 3.9
+  - nodefaults
 dependencies:
   - python>=3.7
   - cftime>=1.1.2
@@ -10,12 +11,13 @@ dependencies:
   - eofs
   - esmpy=*=mpi*  # Ensures MPI works with version of esmpy.
   - ipython
-  - matplotlib
+  - matplotlib-base
   - nc-time-axis
   - netcdf4
   - pip
   - pytest
   - pytest-cov
+  - pytest-xdist
   - scipy
   - xarray>=0.16.1
   - xesmf

climpred/tests/test_HindcastEnsemble_class.py

@@ -108,7 +108,7 @@ def test_inplace(
 def test_dim_input_type(hindcast_hist_obs_1d, dim, call):
     """Test verify and bootstrap for different dim types."""
     kw = dict(iterations=2) if call == "bootstrap" else {}
-    assert getattr(hindcast_hist_obs_1d, call)(
+    assert getattr(hindcast_hist_obs_1d.isel(lead=range(3)), call)(
         metric="rmse", comparison="e2o", dim=dim, alignment="same_verifs", **kw
     )
 
@@ -120,7 +120,7 @@ def test_mean_remove_bias(hindcast_hist_obs_1d, alignment):
     metric = "rmse"
     dim = "init"
     comparison = "e2o"
-    hindcast = hindcast_hist_obs_1d
+    hindcast = hindcast_hist_obs_1d.isel(lead=range(3))
     hindcast._datasets["initialized"].attrs["test"] = "test"
     hindcast._datasets["initialized"]["SST"].attrs["units"] = "test_unit"
     verify_kwargs = dict(
@@ -197,34 +197,6 @@ def test_verify_fails_expected_metric_kwargs(hindcast_hist_obs_1d):
         )
 
 
-def test_verify_m2o_reference(hindcast_hist_obs_1d):
-    """Test that m2o comparison in references work."""
-    hindcast = hindcast_hist_obs_1d
-    # determinstic
-    hindcast.verify(
-        metric="mse",
-        comparison="m2o",
-        dim="init",
-        alignment="same_verif",
-        reference="uninitialized",
-    )
-    hindcast.verify(
-        metric="mse",
-        comparison="m2o",
-        dim="init",
-        alignment="same_verif",
-        reference="persistence",
-    )
-    # probabilistic
-    hindcast.verify(
-        metric="crps",
-        comparison="m2o",
-        reference="uninitialized",
-        dim="member",
-        alignment="same_verif",
-    )
-
-
 def test_calendar_matching_observations(hind_ds_initialized_1d, reconstruction_ds_1d):
     """Tests that error is thrown if calendars mismatch when adding observations."""
     hindcast = HindcastEnsemble(hind_ds_initialized_1d)
@@ -256,7 +228,7 @@ def test_calendar_matching_uninitialized(
 @pytest.mark.parametrize("metric", ["mse", "crps"])
 def test_verify_reference_same_dims(hindcast_hist_obs_1d, metric):
     """Test that verify returns the same dimensionality regardless of reference."""
-    hindcast = hindcast_hist_obs_1d
+    hindcast = hindcast_hist_obs_1d.isel(lead=range(3), init=range(10))
     if metric == "mse":
         comparison = "e2o"
         dim = "init"

climpred/tests/test_PerfectModelEnsemble_class.py

@@ -22,14 +22,12 @@
 ]
 
 references = [
-    [],
     "uninitialized",
     "persistence",
     "climatology",
     ["climatology", "uninitialized", "persistence"],
 ]
 references_ids = [
-    "empty list",
     "uninitialized",
     "persistence",
     "climatology",
@@ -129,18 +127,7 @@ def test_verify_metric_kwargs(perfectModelEnsemble_initialized_control):
     )
 
 
-@pytest.mark.parametrize(
-    "reference",
-    [
-        "uninitialized",
-        ["uninitialized"],
-        "persistence",
-        None,
-        [],
-        "climatology",
-        ["uninitialized", "persistence", "climatology"],
-    ],
-)
+@pytest.mark.parametrize("reference", references, ids=references_ids)
 def test_verify_reference(perfectModelEnsemble_initialized_control, reference):
     """Test that verify works with references given."""
     pm = perfectModelEnsemble_initialized_control.generate_uninitialized()
@@ -248,7 +235,7 @@ def test_HindcastEnsemble_as_PerfectModelEnsemble(hindcast_recon_1d_mm):
     PerfectModelEnsemble."""
     v = "SST"
     alignment = "maximize"
-    hindcast = hindcast_recon_1d_mm
+    hindcast = hindcast_recon_1d_mm.isel(lead=[0, 1])
     assert (
         not hindcast.verify(
             metric="acc", comparison="e2o", dim="init", alignment=alignment
@@ -267,34 +254,13 @@ def test_HindcastEnsemble_as_PerfectModelEnsemble(hindcast_recon_1d_mm):
         .any()
     )
 
-    pm = pm.add_control(
-        init.isel(member=0, lead=0, drop=True)
-        .rename({"init": "time"})
-        .resample(time="1MS")
-        .interpolate("linear")
-    )
-
-    pm = pm.generate_uninitialized()
-    assert (
-        not pm.verify(
-            metric="acc",
-            comparison="m2e",
-            dim=["member", "init"],
-            reference=["uninitialized"],
-        )[v]
-        .isnull()
-        .any()
-    )
-
-    pm.bootstrap(iterations=2, metric="acc", comparison="m2e", dim=["member", "init"])
-
 
 def test_verify_no_need_for_control(PM_da_initialized_1d, PM_da_control_1d):
     """Tests that no error is thrown when no control present
     when calling verify(reference=['uninitialized'])."""
     v = "tos"
     comparison = "m2e"
-    pm = PerfectModelEnsemble(PM_da_initialized_1d).load()
+    pm = PerfectModelEnsemble(PM_da_initialized_1d).isel(lead=[0, 1, 2])
     # verify needs to control
     skill = pm.verify(metric="mse", comparison=comparison, dim="init")
     assert not skill[v].isnull().any()
@@ -336,6 +302,7 @@ def test_verify_no_need_for_control(PM_da_initialized_1d, PM_da_control_1d):
 def test_verify_reference_same_dims(perfectModelEnsemble_initialized_control):
     """Test that verify returns the same dimensionality regardless of reference."""
     pm = perfectModelEnsemble_initialized_control.generate_uninitialized()
+    pm = pm.isel(lead=[0, 1, 2], init=[0, 1, 2])
     metric = "mse"
     comparison = "m2e"
     dim = "init"
@@ -369,9 +336,10 @@ def test_PerfectModel_verify_bootstrap_deterministic(
     perfectModelEnsemble_initialized_control, comparison, metric, dim, reference
 ):
     """
-    Checks that PerfectModel.verify() and PerfectModel.bootstrap() for deterministic metrics is not NaN.
+    Checks that PerfectModel.verify() and PerfectModel.bootstrap() for
+    deterministic metrics is not NaN.
     """
-    pm = perfectModelEnsemble_initialized_control.isel(lead=[0, 1, 2])
+    pm = perfectModelEnsemble_initialized_control.isel(lead=[0, 1, 2], init=range(6))
     if isinstance(reference, str):
         reference = [reference]
     if metric == "contingency":

climpred/tests/test_bootstrap.py

@@ -55,7 +55,7 @@ def test_bootstrap_PM_lazy_results(
     perfectModelEnsemble_initialized_control, chunk, comparison, dim
 ):
     """Test bootstrap_perfect_model works lazily."""
-    pm = perfectModelEnsemble_initialized_control
+    pm = perfectModelEnsemble_initialized_control.isel(lead=range(3))
     if chunk:
         pm = pm.chunk({"lead": 2}).chunk({"time": -1})
     else:
@@ -76,7 +76,7 @@ def test_bootstrap_hindcast_lazy(
     chunk,
 ):
     """Test bootstrap_hindcast works lazily."""
-    he = hindcast_hist_obs_1d
+    he = hindcast_hist_obs_1d.isel(lead=range(3), init=range(10))
     if chunk:
         he = he.chunk({"lead": 2})
     else:
@@ -100,7 +100,7 @@ def test_bootstrap_hindcast_resample_dim(
 ):
     """Test bootstrap_hindcast when resampling member or init and alignment
     same_inits."""
-    hindcast_hist_obs_1d.bootstrap(
+    hindcast_hist_obs_1d.isel(lead=range(3), init=range(10)).bootstrap(
         iterations=ITERATIONS,
         comparison="e2o",
         metric="mse",
@@ -232,6 +232,7 @@ def test_bootstrap_uninit_pm_ensemble_from_control_cftime_annual_identical_da(
 )
 def test_bootstrap_uninit_pm_ensemble_from_control_cftime_all_freq(init, control):
     """Test bootstrap_uninit_pm_ensemble_from_control_cftime for all freq data."""
+    init = init.isel(lead=range(3), init=range(5))
     uninit = bootstrap_uninit_pm_ensemble_from_control_cftime(init, control)
     # lead and member identical
     for d in ["lead", "member"]:
@@ -301,25 +302,6 @@ def test_bootstrap_by_stacking_two_var_dataset(
     assert res["init"].size == res_cf["init"].size
 
 
-@pytest.mark.slow
-@pytest.mark.parametrize("comparison", ["m2o", "e2o"])
-@pytest.mark.parametrize("metric", ["rmse", "pearson_r"])
-@pytest.mark.parametrize("alignment", VALID_ALIGNMENTS)
-def test_bootstrap_hindcast_alignment(
-    hindcast_hist_obs_1d, alignment, metric, comparison
-):
-    """Test bootstrap_hindcast for all alginments when resampling member."""
-    dim = "init" if comparison == "e2o" else ["member", "init"]
-    hindcast_hist_obs_1d.isel(lead=[0, 1, 2]).bootstrap(
-        iterations=ITERATIONS,
-        comparison=comparison,
-        metric=metric,
-        resample_dim="member",
-        alignment=alignment,
-        dim=dim,
-    )
-
-
 def test_bootstrap_hindcast_raises_error(
     hind_da_initialized_1d, hist_da_uninitialized_1d, observations_da_1d
 ):
@@ -366,7 +348,7 @@ def test_resample_size(PM_da_initialized_1d):
 @pytest.mark.parametrize("replace", [True, False])
 def test_resample_iterations_same(PM_da_initialized_1d, chunk, replace):
     """Test that both `resample_iterations` functions yield same result shape."""
-    ds = PM_da_initialized_1d
+    ds = PM_da_initialized_1d.isel(lead=range(3), init=range(5))
     if chunk:
         ds = ds.chunk()
     ds_r_idx = _resample_iterations_idx(ds, ITERATIONS, "member", replace=replace)
@@ -401,7 +383,7 @@ def test_chunk_before_resample_iterations_idx(PM_da_initialized_3d_full):
 @pytest.mark.parametrize("replace", [True, False])
 def test_resample_iterations_dim_max(PM_da_initialized_1d, chunk, replace):
     """Test that both `resample_iterations(dim_max=n)` gives n members."""
-    ds = PM_da_initialized_1d.copy()
+    ds = PM_da_initialized_1d.isel(lead=range(3), init=range(5))
     ds = ds.sel(member=list(ds.member.values) * 2)
     ds["member"] = np.arange(1, 1 + ds.member.size)
     if chunk:

climpred/tests/test_metrics_perfect.py

@@ -30,7 +30,7 @@ def test_PerfectModelEnsemble_constant_forecasts(
 ):
     """Test that PerfectModelEnsemble.verify() returns a perfect score for a perfectly
     identical forecasts."""
-    pe = perfectModelEnsemble_initialized_control.isel(lead=[0, 1, 2])
+    pe = perfectModelEnsemble_initialized_control.isel(lead=[0, 1], init=[0, 1, 2])
     if how == "constant":  # replaces the variable with all 1's
         pe = pe.map(xr.ones_like)
     elif (
@@ -93,7 +93,7 @@ def test_HindcastEnsemble_constant_forecasts(
 ):
     """Test that HindcastEnsemble.verify() returns a perfect score for a perfectly
     identical forecasts."""
-    he = hindcast_hist_obs_1d.isel(lead=[0, 1, 2])
+    he = hindcast_hist_obs_1d.isel(lead=[0, 1], init=range(10))
     if how == "constant":  # replaces the variable with all 1's
         he = he.map(xr.ones_like)
     elif (