Merge branch 'main' into earthkit

environments/environment_wp4.yml

@@ -3,6 +3,7 @@ channels:
 - nodefaults
 dependencies:
 - icclim
+- pint != 0.24
 - pip
 - scipy
 - xarrayMannKendall

notebooks/wp4/cmip6_sea_ice_diagnostics.ipynb

@@ -329,7 +329,9 @@
     "    download.update_request_date(\n",
     "        {\n",
     "            \"cdr_type\": \"cdr\",\n",
-    "            \"origin\": \"eumetsat_osi_saf\",\n",
+    "            \"origin\": \"EUMETSAT OSI SAF\",\n",
+    "            \"sensor\": \"ssmis\",\n",
+    "            \"temporal_aggregation\": \"daily\",\n",
     "            \"variable\": \"all\",\n",
     "            \"version\": \"v2\",\n",
     "        },\n",
@@ -344,7 +346,9 @@
     "    download.update_request_date(\n",
     "        {\n",
     "            \"cdr_type\": \"cdr\",\n",
-    "            \"origin\": \"esa_cci\",\n",
+    "            \"origin\": \"ESA CCI\",\n",
+    "            \"sensor\": \"amsr\",\n",
+    "            \"temporal_aggregation\": \"daily\",\n",
     "            \"variable\": \"all\",\n",
     "            \"version\": \"v2\",\n",
     "        },\n",
@@ -486,7 +490,9 @@
     "        \"version\": \"v2\",\n",
     "        \"variable\": \"all\",\n",
     "        \"format\": \"zip\",\n",
-    "        \"origin\": \"esa_cci\",\n",
+    "        \"origin\": \"ESA CCI\",\n",
+    "        \"sensor\": \"amsr\",\n",
+    "        \"temporal_aggregation\": \"daily\",\n",
     "        \"cdr_type\": \"cdr\",\n",
     "        \"year\": \"2002\",\n",
     "        \"month\": \"06\",\n",

notebooks/wp4/cmip6_sea_ice_evaluation.ipynb

@@ -339,7 +339,9 @@
     "        \"version\": \"v2\",\n",
     "        \"variable\": \"all\",\n",
     "        \"format\": \"zip\",\n",
-    "        \"origin\": \"esa_cci\",\n",
+    "        \"origin\": \"ESA CCI\",\n",
+    "        \"sensor\": \"amsr\",\n",
+    "        \"temporal_aggregation\": \"daily\",\n",
     "        \"cdr_type\": \"cdr\",\n",
     "        \"year\": \"2002\",\n",
     "        \"month\": \"06\",\n",
@@ -386,13 +388,23 @@
     "    }\n",
     "    satellite_requests = {\n",
     "        \"ESA-CCI\": download.update_request_date(\n",
-    "            common_request | {\"origin\": \"esa_cci\"},\n",
+    "            common_request\n",
+    "            | {\n",
+    "                \"origin\": \"ESA CCI\",\n",
+    "                \"sensor\": \"amsr\",\n",
+    "                \"temporal_aggregation\": \"daily\",\n",
+    "            },\n",
     "            start=f\"{max(year_start, 2002)}-01\",\n",
     "            stop=f\"{min(year_stop, 2017)}-12\",\n",
     "            stringify_dates=True,\n",
     "        ),\n",
     "        \"EUMETSAT-OSI-SAF\": download.update_request_date(\n",
-    "            common_request | {\"origin\": \"eumetsat_osi_saf\"},\n",
+    "            common_request\n",
+    "            | {\n",
+    "                \"origin\": \"EUMETSAT OSI SAF\",\n",
+    "                \"sensor\": \"ssmis\",\n",
+    "                \"temporal_aggregation\": \"daily\",\n",
+    "            },\n",
     "            start=f\"{max(year_start, 1979)}-01\",\n",
     "            stop=f\"{min(year_stop, 2015)}-12\",\n",
     "            stringify_dates=True,\n",
@@ -516,7 +528,9 @@
     "        \"version\": \"v2\",\n",
     "        \"variable\": \"all\",\n",
     "        \"format\": \"zip\",\n",
-    "        \"origin\": \"esa_cci\",\n",
+    "        \"origin\": \"ESA CCI\",\n",
+    "        \"sensor\": \"amsr\",\n",
+    "        \"temporal_aggregation\": \"daily\",\n",
     "        \"cdr_type\": \"cdr\",\n",
     "        \"year\": \"2002\",\n",
     "        \"month\": \"06\",\n",
@@ -699,7 +713,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

notebooks/wp5/FIRE_burned_area_trends.ipynb

@@ -162,7 +162,7 @@
     "ds.rio.write_crs(crs, inplace=True)\n",
     "\n",
     "regions = regionmask.from_geopandas(gdf, names=\"NUTS_NAME\")\n",
-    "mask = regions.mask(ds[\"longitude\"], ds[\"latitude\"])"
+    "mask = regions.mask_3D(ds[\"longitude\"], ds[\"latitude\"])"
    ]
   },
   {
@@ -180,8 +180,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "da_grouped = da.groupby(mask.rename(\"region\")).sum((\"latitude\", \"longitude\", \"month\"))\n",
-    "da_grouped = da_grouped.assign_coords(region=regions.names).compute()\n",
+    "da_grouped = da.where(mask).sum((\"latitude\", \"longitude\", \"month\"))\n",
+    "da_grouped = da_grouped.assign_coords(region=mask[\"names\"]).compute()\n",
     "\n",
     "df_dict = {\"slope\": [], \"intercept\": [], \"tau\": [], \"p-value\": []}\n",
     "for region in gdf[\"NUTS_NAME\"]:\n",
@@ -321,7 +321,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

notebooks/wp5/LUCL_land_cover_classification_reliability.ipynb

@@ -113,7 +113,7 @@
     "    transform_func=utils.regionalise,\n",
     "    transform_func_kwargs={\"lon_slice\": lon_slice, \"lat_slice\": lat_slice},\n",
     ")\n",
-    "ds = ds.assign_coords(year=ds[\"time\"].dt.year).swap_dims(time=\"year\").drop(\"time\")"
+    "ds = ds.assign_coords(year=ds[\"time\"].dt.year).swap_dims(time=\"year\").drop_vars(\"time\")"
    ]
   },
   {

notebooks/wp5/LULC_land_cover_urbanization_trends.ipynb

@@ -113,7 +113,7 @@
     "    transform_func=utils.regionalise,\n",
     "    transform_func_kwargs={\"lon_slice\": lon_slice, \"lat_slice\": lat_slice},\n",
     ")\n",
-    "ds = ds.assign_coords(year=ds[\"time\"].dt.year).swap_dims(time=\"year\").drop(\"time\")"
+    "ds = ds.assign_coords(year=ds[\"time\"].dt.year).swap_dims(time=\"year\").drop_vars(\"time\")"
    ]
   },
   {

notebooks/wp5/era5_cerra_extremes.ipynb

@@ -31,6 +31,7 @@
     "import matplotlib.pyplot as plt\n",
     "import xarray as xr\n",
     "from c3s_eqc_automatic_quality_control import download, plot, utils\n",
+    "from cartopy.mpl.gridliner import Gridliner\n",
     "\n",
     "plt.style.use(\"seaborn-v0_8-notebook\")"
    ]
@@ -227,7 +228,8 @@
     "    plot.projected_map(da, projection=projection, vmin=vmin, vmax=vmax, cmap=\"turbo\")\n",
     "    plot_obj = plt.title(f\"{collection_id.split('-')[1].upper()} {title}\")\n",
     "    plot_obj.axes.set_extent(extent)\n",
-    "    for gl in plot_obj.axes._gridliners:\n",
+    "    gridliners = [a for a in plot_obj.axes.artists if isinstance(a, Gridliner)]\n",
+    "    for gl in gridliners:\n",
     "        gl.x_inline = False\n",
     "        gl.xlabel_style = {\"rotation\": 0}\n",
     "    plt.show()"
@@ -284,7 +286,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

notebooks/wp5/glacier_mass_balance_maps.ipynb

@@ -28,6 +28,7 @@
     "import matplotlib.colors as mcolors\n",
     "import matplotlib.pyplot as plt\n",
     "import xarray as xr\n",
+    "import xskillscore as xs\n",
     "from c3s_eqc_automatic_quality_control import download\n",
     "\n",
     "plt.style.use(\"seaborn-v0_8-notebook\")"
@@ -97,7 +98,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def compute_sum_uncertainty_slope_and_acceleration(ds):\n",
+    "def compute_coeff_and_pvalue(cumsum, degree):\n",
+    "    coeff = cumsum.polyfit(\"time\", degree)\n",
+    "    (fit,) = xr.polyval(cumsum[\"time\"], coeff).data_vars.values()\n",
+    "    p_value = xs.pearson_r_p_value(cumsum.chunk(time=-1), fit.chunk(time=-1), \"time\")\n",
+    "    return coeff, p_value\n",
+    "\n",
+    "\n",
+    "def compute_time_statistics(ds):\n",
     "    with xr.set_options(keep_attrs=True):\n",
     "        cumsum = ds[\"Glacier\"].cumsum(\"time\").drop_vars(\"time\")\n",
     "        ds[\"Uncertainty\"] = (ds[\"Uncertainty\"] ** 2).sum(\"time\") ** (1 / 2)\n",
@@ -108,22 +116,35 @@
     "        da.attrs[\"long_name\"] = f\"Sum of {da.attrs['long_name']}\"\n",
     "\n",
     "    # Linear\n",
-    "    ds[\"Slope\"] = cumsum.polyfit(\"time\", 1)[\"polyfit_coefficients\"].sel(degree=1)\n",
+    "    coeff, p_value = compute_coeff_and_pvalue(cumsum, 1)\n",
+    "    ds[\"Slope\"] = coeff[\"polyfit_coefficients\"].sel(degree=1)\n",
     "    ds[\"Slope\"].attrs = {\n",
     "        \"long_name\": f\"Trend of {da.attrs['long_name']}\",\n",
     "        \"units\": f\"{da.attrs['units']} yr$^{-1}$\",\n",
     "    }\n",
+    "    ds[\"Pvalue1\"] = p_value\n",
+    "    ds[\"Pvalue1\"].attrs = {\n",
+    "        \"long_name\": f\"Linear p-value of {da.attrs['long_name']}\",\n",
+    "    }\n",
     "\n",
     "    # Quadratic\n",
-    "    ds[\"Acceleration\"] = 2 * cumsum.polyfit(\"time\", 2)[\"polyfit_coefficients\"].sel(\n",
-    "        degree=2\n",
-    "    )\n",
+    "    coeff, p_value = compute_coeff_and_pvalue(cumsum, 2)\n",
+    "    ds[\"Acceleration\"] = 2 * coeff[\"polyfit_coefficients\"].sel(degree=2)\n",
     "    ds[\"Acceleration\"].attrs = {\n",
     "        \"long_name\": f\"Quadratic Trend of {da.attrs['long_name']}\",\n",
     "        \"units\": f\"{da.attrs['units']} yr$^{-2}$\",\n",
     "    }\n",
+    "    ds[\"Pvalue2\"] = p_value\n",
+    "    ds[\"Pvalue2\"].attrs = {\n",
+    "        \"long_name\": f\"Quadratic p-value of {da.attrs['long_name']}\",\n",
+    "    }\n",
+    "\n",
+    "    return ds\n",
     "\n",
-    "    return ds"
+    "\n",
+    "def compute_spatial_statistics(ds):\n",
+    "    with xr.set_options(keep_attrs=True):\n",
+    "        return ds.sum((\"latitude\", \"longitude\"))"
    ]
   },
   {
@@ -141,16 +162,25 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "chunks = {\"hydrological_year\": 1}\n",
     "ds = download.download_and_transform(\n",
     "    collection_id,\n",
     "    request,\n",
-    "    chunks={\"hydrological_year\": 1},\n",
-    "    transform_func=compute_sum_uncertainty_slope_and_acceleration,\n",
+    "    chunks=chunks,\n",
+    "    transform_func=compute_time_statistics,\n",
     "    transform_chunks=False,\n",
     ")\n",
+    "ds_timeseries = download.download_and_transform(\n",
+    "    collection_id,\n",
+    "    request,\n",
+    "    chunks=chunks,\n",
+    "    transform_func=compute_spatial_statistics,\n",
+    ")\n",
+    "\n",
     "# Customize some attributes\n",
-    "for da in ds.data_vars.values():\n",
-    "    da.attrs[\"long_name\"] = da.attrs[\"long_name\"].replace(\"_\", \" \").title()"
+    "for obj in (ds, ds_timeseries):\n",
+    "    for da in obj.data_vars.values():\n",
+    "        da.attrs[\"long_name\"] = da.attrs[\"long_name\"].replace(\"_\", \" \").title()"
    ]
   },
   {
@@ -252,19 +282,45 @@
     "        \"label\": \"Mass change trend (Gt yr$^{-1}$)\",\n",
     "        \"title\": \"Linear trends (slopes) of the mass changes of the glaciers.\",\n",
     "    },\n",
+    "    \"Pvalue1\": {\n",
+    "        \"title\": \"Linear p-value.\",\n",
+    "    },\n",
     "    \"Acceleration\": {\n",
     "        \"cmap\": \"coolwarm_r\",\n",
     "        \"vmin\": -0.005,\n",
     "        \"vmax\": +0.005,\n",
     "        \"label\": \"Total mass change error (Gt yr$^{-2}$)\",\n",
     "        \"title\": \"Quadratic trends (accelerations) of the mass changes of the glaciers.\",\n",
     "    },\n",
+    "    \"Pvalue2\": {\n",
+    "        \"title\": \"Quadratics p-value.\",\n",
+    "    },\n",
     "}\n",
-    "\n",
     "for var_name, kwargs in column_kwargs.items():\n",
     "    ax = plot_map(gdf, var_name, **kwargs)\n",
     "    plt.show()"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17",
+   "metadata": {},
+   "source": [
+    "## Plot timeseries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for da in ds_timeseries.data_vars.values():\n",
+    "    da.plot()\n",
+    "    plt.grid()\n",
+    "    plt.show()"
+   ]
   }
  ],
  "metadata": {
@@ -283,7 +339,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,