remove outliers

bopen · Mar 26, 2024 · be96a04 · be96a04
1 parent f404923
commit be96a04
Showing 1 changed file with 18 additions and 11 deletions.
diff --git a/notebooks/wp5/satellite_sea_ice_conc_climate_change.ipynb b/notebooks/wp5/satellite_sea_ice_conc_climate_change.ipynb
@@ -52,8 +52,8 @@
    "outputs": [],
    "source": [
     "# Time\n",
-    "start_year = 1979\n",
-    "stop_year = 2023\n",
+    "year_start = 1979\n",
+    "year_stop = 2023\n",
     "\n",
     "# Conc threshold for calculating extent\n",
     "sic_threshold = 15"
@@ -86,22 +86,22 @@
     "    # CDR\n",
     "    \"EUMETSAT-OSI-SAF (CDR)\": download.update_request_date(\n",
     "        conc_request | {\"origin\": \"eumetsat_osi_saf\"},\n",
-    "        start=f\"{max(start_year, 1979)}-01\",\n",
-    "        stop=f\"{min(stop_year, 2015)}-12\",\n",
+    "        start=f\"{max(year_start, 1979)}-01\",\n",
+    "        stop=f\"{min(year_stop, 2015)}-12\",\n",
     "        stringify_dates=True,\n",
     "    ),\n",
     "    # interim CDR for later years\n",
     "    \"EUMETSAT-OSI-SAF (ICDR)\": download.update_request_date(\n",
     "        conc_request | {\"cdr_type\": \"icdr\", \"origin\": \"eumetsat_osi_saf\"},\n",
-    "        start=f\"{max(start_year, 2016)}-01\",\n",
-    "        stop=f\"{min(stop_year, 2023)}-12\",\n",
+    "        start=f\"{max(year_start, 2016)}-01\",\n",
+    "        stop=f\"{min(year_stop, 2023)}-12\",\n",
     "        stringify_dates=True,\n",
     "    ),\n",
     "    # only CDR available for ESA-CCI\n",
     "    \"ESA-CCI (CDR)\": download.update_request_date(\n",
     "        conc_request | {\"origin\": \"esa_cci\"},\n",
-    "        start=f\"{max(start_year, 2002)}-01\",\n",
-    "        stop=f\"{min(stop_year, 2017)}-12\",\n",
+    "        start=f\"{max(year_start, 2002)}-01\",\n",
+    "        stop=f\"{min(year_stop, 2017)}-12\",\n",
     "        stringify_dates=True,\n",
     "    ),\n",
     "}"
@@ -230,11 +230,19 @@
     "    return ds.set_index(time=new_dims).unstack(\"time\")\n",
     "\n",
     "\n",
-    "def compute_yearly_extremes(da, reduction, min_samples=150):\n",
+    "def compute_yearly_extremes(da, reduction, min_samples=150, remove_outliers=True):\n",
     "    grouped = da.groupby(\"time.year\")\n",
     "    mask = grouped.count() > min_samples\n",
     "    da = getattr(grouped, reduction)(keep_attrs=True)\n",
-    "    return da.where(mask.compute(), drop=True)\n",
+    "    if remove_outliers:\n",
+    "        da = da.chunk(year=-1)\n",
+    "        q1 = da.quantile(1 / 4, \"year\")\n",
+    "        q3 = da.quantile(3 / 4, \"year\")\n",
+    "        delta = 1.5 * (q3 - q1)\n",
+    "        mask &= (da >= q1 - delta) & (da <= q3 + delta)\n",
+    "    da = da.where(mask.compute(), drop=True)\n",
+    "    da.attrs[\"long_name\"] = \" \".join([reduction.title(), da.attrs[\"long_name\"]])\n",
+    "    return da\n",
     "\n",
     "\n",
     "def plot_against_dayofyear(\n",
@@ -293,7 +301,6 @@
     "for region, da_region in ds[\"extent\"].groupby(\"region\"):\n",
     "    for reduction in (\"min\", \"max\"):\n",
     "        da = compute_yearly_extremes(da_region, reduction)\n",
-    "        da.attrs[\"long_name\"] = \" \".join([reduction.title(), da.attrs[\"long_name\"]])\n",
     "        da.plot(hue=\"product\", marker=\"^\")\n",
     "        plt.grid()\n",
     "        plt.show()"