Add bootstrapping, data validation, trace writing, and others

- Read and write scaling parameters from/to trace file
- Add validate function/hook to variables and check for units
- Check input metadata
- Pin random seed
- Store replaced NaN, Inf, -Inf values in output
- Add bootstrapping
- Allow selection of cells
- Add traces output as well as reading from traces instead of fitting

Co-authored-by: Sven Willner <[email protected]>
Co-authored-by: Robert Gieseke <[email protected]>

attrici/cli.py

@@ -5,7 +5,6 @@
 from attrici.commands import (
     detrend,
     merge_output,
-    merge_traces,
     postprocess_tas,
     preprocess_tas,
     ssa,
@@ -33,7 +32,6 @@ def main():
 
     detrend.add_parser(subparsers)
     merge_output.add_parser(subparsers)
-    merge_traces.add_parser(subparsers)
     preprocess_tas.add_parser(subparsers)
     postprocess_tas.add_parser(subparsers)
     ssa.add_parser(subparsers)

attrici/commands/detrend.py

@@ -42,6 +42,30 @@ def iso_date(argument_value):
         raise argparse.ArgumentTypeError(e)
 
 
+def lat_lons(argument_value):
+    """
+    Try parsing `argument_value` from a semicolon-separated string of lat,lon tuples
+    into a list of tuples. Used as an argument type for `--cells`.
+
+    Parameters
+    ----------
+    argument_value : str
+            The string value of the argument
+
+    Returns
+    -------
+    list of tuple
+            A list of lat,lon tuples
+    """
+    try:
+        return [
+            (float(lat), float(lon))
+            for lat, lon in [pair.split(",") for pair in argument_value.split(";")]
+        ]
+    except ValueError as e:
+        raise argparse.ArgumentTypeError(e)
+
+
 def add_parser(subparsers):
     parser = subparsers.add_parser(
         "detrend",
@@ -64,11 +88,25 @@ def add_parser(subparsers):
         help="(SSA-smoothed) Global Mean Temperature file",
         required=True,
     )
+    group.add_argument(
+        "--gmt-variable",
+        type=str,
+        help="(SSA-smoothed) Global Mean Temperature variable name",
+        default=Config.__dataclass_fields__["gmt_variable"].default,
+    )
     group.add_argument("--input-file", type=Path, help="Input file", required=True)
     group.add_argument("--mask-file", type=Path, help="Mask file")
     group.add_argument(
         "--variable", type=str, help="Variable to detrend", required=True
     )
+    group.add_argument("--trace-file", type=Path, help="Trace file")
+    group.add_argument("--fit-only", action="store_true", help="Only fit the model")
+    group.add_argument(
+        "--cells",
+        type=lat_lons,
+        help="Semicolon-separated lat,lon tuples to process,"
+        " otherwise all cells are processed",
+    )
 
     # output
     group = parser.add_argument_group(title="Output")
@@ -81,6 +119,7 @@ def add_parser(subparsers):
     group.add_argument(
         "--overwrite", action="store_true", help="Overwrite existing files"
     )
+    group.add_argument("--write-trace", action="store_true", help="Save trace to file")
 
     # run parameters
     group = parser.add_argument_group(title="Run parameters")
@@ -91,6 +130,12 @@ def add_parser(subparsers):
         help="Number of modes for fourier series of model (either one integer, used for"
         " all four series, or four comma-separated integers)",
     )
+    group.add_argument(
+        "--bootstrap-sample-count",
+        type=int,
+        default=Config.__dataclass_fields__["bootstrap_sample_count"].default,
+        help="Number of bootstrap samples",
+    )
     group.add_argument("--progressbar", action="store_true", help="Show progress bar")
     group.add_argument(
         "--report-variables",

attrici/commands/merge_output.py

@@ -7,7 +7,7 @@
 
 def run(args):
     ds = xr.open_mfdataset(
-        str(args.directory / "*/ts_*.nc"),
+        str(args.directory / "*/*.nc"),
         parallel=True,
         engine="h5netcdf",
         # for parallel=True, only engine="h5netcdf" seems to work for now
@@ -19,13 +19,13 @@ def run(args):
 def add_parser(subparsers):
     parser = subparsers.add_parser(
         "merge-output",
-        help="Merge detrended output",
+        help="Merge detrended output or trace files",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(
-        "directory", type=Path, help="Directory containing detrended output timeseries"
+        "directory",
+        type=Path,
+        help="Directory containing detrended output timeseries or trace files",
     )
-    parser.add_argument("variable", type=str, help="Variable to merge")
-    parser.add_argument("output_variable", type=str, help="Variable name in output")
     parser.add_argument("output_filename", type=Path, help="Merged output file")
     parser.set_defaults(func=run)