Fix for avoiding serialization error when having datetimes larger tha…

…n 2263

src/ert/analysis/_es_update.py

@@ -305,13 +305,15 @@ def _get_obs_and_measure_data(
     for obs_key, obs_active_list in selected_observations:
         observation = observations[obs_key]
         group = observation.attrs["response"]
+
         if obs_active_list:
             index = observation.coords.to_index()[obs_active_list]
             sub_selection = {
                 name: list(set(index.get_level_values(name))) for name in index.names
             }
             observation = observation.sel(sub_selection)
         ds = source_fs.load_responses(group, tuple(iens_active_index))
+
         try:
             filtered_ds = observation.merge(ds, join="left")
         except KeyError as e:

src/ert/config/summary_config.py

@@ -43,8 +43,13 @@ def read_from_file(self, run_path: str, iens: int) -> xr.Dataset:
                     f"{last} from: {run_path}/{filename}.UNSMRY"
                 )
 
+        # Internally we need to use strings to represent time because numpy.datetime64[ns] only allows time until 2262 (for now)
+        # Convert the time_map from datetime.datetime to string
+        time_map = [datetime.isoformat(t, timespec="microseconds") for t in time_map]
+
         ds = xr.Dataset(
             {"values": (["name", "time"], data)},
             coords={"time": time_map, "name": keys},
         )
+
         return ds.drop_duplicates("time")

src/ert/data/_measured_data.py

@@ -115,6 +115,7 @@ def _get_data(
                     raise ResponseError(_msg)
             except KeyError as e:
                 raise ResponseError(_msg) from e
+
             ds = obs.merge(
                 response,
                 join="left",

src/ert/storage/local_ensemble.py

@@ -378,6 +378,7 @@ def load_responses(
             ds = xr.open_dataset(input_path, engine="scipy")
             loaded.append(ds)
         response = xr.combine_nested(loaded, concat_dim="realization")
+
         assert isinstance(response, xr.Dataset)
         return response
 
@@ -419,7 +420,9 @@ def load_all_summary_data(
         summary_keys = self.get_summary_keyset()
 
         try:
-            df = self.load_responses("summary", tuple(realizations)).to_dataframe()
+            df = self.load_responses("summary", tuple(realizations)).to_dataframe(
+                ["time", "name", "realization"]
+            )
         except (ValueError, KeyError):
             return pd.DataFrame()
 

src/ert/storage/local_experiment.py

@@ -130,11 +130,20 @@ def response_configuration(self) -> Dict[str, ResponseConfig]:
 
     @property
     def observations(self) -> Dict[str, xr.Dataset]:
-        observations = list(self.mount_point.glob("observations/*"))
-        return {
-            observation.name: xr.open_dataset(observation, engine="scipy")
-            for observation in observations
-        }
+        observations = {}
+        for observation in list(self.mount_point.glob("observations/*")):
+            ds = xr.open_dataset(observation, engine="scipy")
+            if "time" in ds.coords:
+                # Internally we need to use strings to represent time because numpy.datetime64[ns] only allows time until 2262 (for now)
+                # Convert the time for the observation from numpy.datetime64[ns] (nanosecords) to datetime64[us] (microseconds) then to string
+                ds.coords["time"] = [
+                    t
+                    for t in ds.coords["time"]
+                    .values.astype("datetime64[us]")
+                    .astype(str)
+                ]
+            observations[observation.name] = ds
+        return observations
 
 
 class LocalExperimentAccessor(LocalExperimentReader):

tests/unit_tests/data/test_integration_data.py

@@ -48,9 +48,10 @@ def test_summary_obs(create_measured_data):
     summary_obs.remove_inactive_observations()
     assert all(summary_obs.data.columns.get_level_values("data_index").values == [71])
     # Only one observation, we check the key_index is what we expect:
-    assert summary_obs.data.columns.get_level_values("key_index").values[
-        0
-    ] == np.datetime64("2011-12-21")
+    assert (
+        summary_obs.data.columns.get_level_values("key_index").values[0]
+        == "2011-12-21T00:00:00.000000"
+    )
 
 
 @pytest.mark.filterwarnings("ignore::ert.config.ConfigWarning")
@@ -106,7 +107,8 @@ def test_gen_obs_and_summary(create_measured_data):
 
 
 def test_gen_obs_and_summary_index_range(create_measured_data):
-    df = create_measured_data(["WPR_DIFF_1", "FOPR"], [[800], [datetime(2010, 4, 20)]])
+    t = datetime.isoformat(datetime(2010, 4, 20), timespec="microseconds")
+    df = create_measured_data(["WPR_DIFF_1", "FOPR"], [[800], [t]])
     df.remove_inactive_observations()
 
     assert df.data.columns.get_level_values(0).to_list() == [

tests/unit_tests/snapshots/test_libres_facade/test_summary_collector/0/summary_collector_1.csv

@@ -1,5 +1,5 @@
 Realization,Date,"BPR:1,3,8","BPR:5,5,5",FGIP,FGIPH,FGOR,FGORH,FGPR,FGPRH,FGPT,FGPTH,FOIP,FOIPH,FOPR,FOPRH,FOPT,FOPTH,FWCT,FWCTH,FWIP,FWIPH,FWPR,FWPRH,FWPT,FWPTH,WGOR:OP1,WGOR:OP2,WGORH:OP1,WGORH:OP2,WGPR:OP1,WGPR:OP2,WGPRH:OP1,WGPRH:OP2,WOPR:OP1,WOPR:OP2,WOPRH:OP1,WOPRH:OP2,WWCT:OP1,WWCT:OP2,WWCTH:OP1,WWCTH:OP2,WWPR:OP1,WWPR:OP2,WWPRH:OP1,WWPRH:OP2
-0,2010-01-10,0.9996,0.9996,2499.4473,2499.9956,1.0,1.0,0.0557,0.0012,0.5528,0.0044,1999.4462,1999.994,0.056,0.0017,0.5538,0.0059,0.1776,0.0002,2249.4492,2249.9998,0.0551,0.0,0.5507,1e-04,1.0,1.0,1.0,1.0,0.0557,0.0,0.0006,0.0006,0.056,0.0,0.0008,0.0008,0.3552,0.0,1e-04,0.0002,0.0551,0.0,0.0,0.0
-1,2010-01-10,0.9996,0.9996,2499.8467,2499.9956,1.0,1.0,0.0157,0.0012,0.1533,0.0044,1999.8458,1999.994,0.016,0.0017,0.1542,0.0059,0.0657,0.0002,2249.8489,2249.9998,0.0151,0.0,0.1512,1e-04,1.0,1.0,1.0,1.0,0.0,0.0157,0.0006,0.0006,0.0,0.016,0.0008,0.0008,0.0,0.1314,1e-04,0.0002,0.0,0.0151,0.0,0.0
-2,2010-01-10,0.9996,0.9996,2500.0,2499.9956,1.0,1.0,0.0,0.0012,0.0,0.0044,2000.0,1999.994,0.0,0.0017,0.0,0.0059,0.0,0.0002,2250.0,2249.9998,0.0,0.0,0.0,1e-04,1.0,1.0,1.0,1.0,0.0,0.0,0.0006,0.0006,0.0,0.0,0.0008,0.0008,0.0,0.0,1e-04,0.0002,0.0,0.0,0.0,0.0
-3,2010-01-10,0.9996,0.9996,2497.1733,2499.9956,0.9994,1.0,0.2835,0.0012,2.8267,0.0044,1997.1716,1999.994,0.284,0.0017,2.8285,0.0059,0.4825,0.0002,2247.1775,2249.9998,0.2823,0.0,2.8224,1e-04,1.0,0.9987,1.0,1.0,0.0879,0.1956,0.0006,0.0006,0.0882,0.1958,0.0008,0.0008,0.4661,0.4989,1e-04,0.0002,0.0873,0.195,0.0,0.0
+0,2010-01-10T00:00:00.000000,0.9996,0.9996,2499.4473,2499.9956,1.0,1.0,0.0557,0.0012,0.5528,0.0044,1999.4462,1999.994,0.056,0.0017,0.5538,0.0059,0.1776,0.0002,2249.4492,2249.9998,0.0551,0.0,0.5507,1e-04,1.0,1.0,1.0,1.0,0.0557,0.0,0.0006,0.0006,0.056,0.0,0.0008,0.0008,0.3552,0.0,1e-04,0.0002,0.0551,0.0,0.0,0.0
+1,2010-01-10T00:00:00.000000,0.9996,0.9996,2499.8467,2499.9956,1.0,1.0,0.0157,0.0012,0.1533,0.0044,1999.8458,1999.994,0.016,0.0017,0.1542,0.0059,0.0657,0.0002,2249.8489,2249.9998,0.0151,0.0,0.1512,1e-04,1.0,1.0,1.0,1.0,0.0,0.0157,0.0006,0.0006,0.0,0.016,0.0008,0.0008,0.0,0.1314,1e-04,0.0002,0.0,0.0151,0.0,0.0
+2,2010-01-10T00:00:00.000000,0.9996,0.9996,2500.0,2499.9956,1.0,1.0,0.0,0.0012,0.0,0.0044,2000.0,1999.994,0.0,0.0017,0.0,0.0059,0.0,0.0002,2250.0,2249.9998,0.0,0.0,0.0,1e-04,1.0,1.0,1.0,1.0,0.0,0.0,0.0006,0.0006,0.0,0.0,0.0008,0.0008,0.0,0.0,1e-04,0.0002,0.0,0.0,0.0,0.0
+3,2010-01-10T00:00:00.000000,0.9996,0.9996,2497.1733,2499.9956,0.9994,1.0,0.2835,0.0012,2.8267,0.0044,1997.1716,1999.994,0.284,0.0017,2.8285,0.0059,0.4825,0.0002,2247.1775,2249.9998,0.2823,0.0,2.8224,1e-04,1.0,0.9987,1.0,1.0,0.0879,0.1956,0.0006,0.0006,0.0882,0.1958,0.0008,0.0008,0.4661,0.4989,1e-04,0.0002,0.0873,0.195,0.0,0.0

tests/unit_tests/snapshots/test_libres_facade/test_summary_collector/0/summary_collector_2.csv

@@ -1,5 +1,5 @@
 Realization,Date,WWCT:OP1,WWCT:OP2
-0,2010-01-10,0.35516918,0.0
-1,2010-01-10,0.0,0.1313817
-2,2010-01-10,0.0,0.0
-3,2010-01-10,0.46613,0.49889395
+0,2010-01-10T00:00:00.000000,0.35516918,0.0
+1,2010-01-10T00:00:00.000000,0.0,0.1313817
+2,2010-01-10T00:00:00.000000,0.0,0.0
+3,2010-01-10T00:00:00.000000,0.46613,0.49889395

tests/unit_tests/snapshots/test_libres_facade/test_summary_collector/0/summary_collector_3.csv

@@ -1,5 +1,5 @@
 Realization,Date,WWCT:OP1,WWCT:OP2
-4,2010-01-10,0.19479288,0.0
-4,2010-01-20,0.1951685,0.0
-4,2010-01-30,0.19581304,0.0
-4,2010-02-09,0.19672726,0.0
+4,2010-01-10T00:00:00.000000,0.19479288,0.0
+4,2010-01-20T00:00:00.000000,0.1951685,0.0
+4,2010-01-30T00:00:00.000000,0.19581304,0.0
+4,2010-02-09T00:00:00.000000,0.19672726,0.0

tests/unit_tests/snapshots/test_libres_facade/test_summary_data_verify_indices_and_values/0/summary_head.csv

@@ -1,8 +1,8 @@
 ,FOPR,FOPR,FOPR,FOPR,FOPR
 Realization,0,1,2,3,4
 Date,,,,,
-2010-01-10,0.055961154,0.0159828,0.0,0.2839923,0.02509654
-2010-01-20,0.05905991,0.01898478,0.0,0.2900899,0.028275477
-2010-01-30,0.064337626,0.024110613,0.0,0.30051348,0.03370005
-2010-02-09,0.07174286,0.031328063,0.0009306805,0.31520578,0.041331254
-2010-02-19,0.081208415,0.04059234,0.009585856,0.33403203,0.0511149
+2010-01-10T00:00:00.000000,0.055961154,0.0159828,0.0,0.2839923,0.02509654
+2010-01-20T00:00:00.000000,0.05905991,0.01898478,0.0,0.2900899,0.028275477
+2010-01-30T00:00:00.000000,0.064337626,0.024110613,0.0,0.30051348,0.03370005
+2010-02-09T00:00:00.000000,0.07174286,0.031328063,0.0009306805,0.31520578,0.041331254
+2010-02-19T00:00:00.000000,0.081208415,0.04059234,0.009585856,0.33403203,0.0511149

tests/unit_tests/snapshots/test_libres_facade/test_summary_data_verify_indices_and_values/0/summary_tail.csv

@@ -1,8 +1,8 @@
 ,FOPR,FOPR,FOPR,FOPR,FOPR
 Realization,0,1,2,3,4
 Date,,,,,
-2015-05-14,0.05978195,0.028228229,0.0,0.2988855,0.03184779
-2015-05-24,0.06022726,0.02732692,0.0,0.2993037,0.032080103
-2015-06-03,0.060678575,0.026679888,0.0,0.29959962,0.032258306
-2015-06-13,0.06101494,0.026284758,0.0,0.2997273,0.03237228
-2015-06-23,0.061153173,0.026137633,0.0,0.29976672,0.032415867
+2015-05-14T00:00:00.000000,0.05978195,0.028228229,0.0,0.2988855,0.03184779
+2015-05-24T00:00:00.000000,0.06022726,0.02732692,0.0,0.2993037,0.032080103
+2015-06-03T00:00:00.000000,0.060678575,0.026679888,0.0,0.29959962,0.032258306
+2015-06-13T00:00:00.000000,0.06101494,0.026284758,0.0,0.2997273,0.03237228
+2015-06-23T00:00:00.000000,0.061153173,0.026137633,0.0,0.29976672,0.032415867

tests/unit_tests/snapshots/test_summary_response/test_load_summary_response_restart_not_zero/summary_restart

@@ -1,7 +1,7 @@
 Realization,Date,FGIP,FGIR,FGIRH,FGIT,FGITH,FGLIR,FGLR,FGOR,FGORH,FGPP,FGPR,FGPRF,FGPRH,FGPRS,FGPT
-0,2020-07-02,5737580500.0,0.0,0.0,0.0,0.0,0.0,68.796104,146.78568,146.78568,2777220.0,853880.56,165187.05,853880.56,688693.56,1168540500.0
-0,2020-08-01,5711617500.0,0.0,0.0,0.0,0.0,0.0,68.93971,151.33902,151.33902,2757425.0,866019.94,192419.36,866019.94,673600.56,1194503800.0
-0,2020-09-01,5684979000.0,0.0,0.0,0.0,0.0,0.0,68.258804,154.72658,154.72658,2712226.0,856132.9,208552.92,856132.9,647579.94,1221141900.0
-0,2020-10-01,5659591700.0,0.0,0.0,0.0,0.0,0.0,66.956535,156.4753,156.4753,2654457.8,837970.5,214351.67,837970.5,623618.9,1246529300.0
-0,2020-11-01,5634260500.0,0.0,0.0,0.0,0.0,0.0,64.80198,155.93571,155.93571,2580463.0,809597.94,207660.8,809597.94,601937.1,1271860400.0
-0,2020-12-01,5610946000.0,0.0,0.0,0.0,0.0,0.0,62.265244,153.91632,153.91632,2505382.5,777148.44,193741.4,777148.44,583407.06,1295174900.0
+0,2020-07-02T00:00:00.000000,5737580500.0,0.0,0.0,0.0,0.0,0.0,68.796104,146.78568,146.78568,2777220.0,853880.56,165187.05,853880.56,688693.56,1168540500.0
+0,2020-08-01T00:00:00.000000,5711617500.0,0.0,0.0,0.0,0.0,0.0,68.93971,151.33902,151.33902,2757425.0,866019.94,192419.36,866019.94,673600.56,1194503800.0
+0,2020-09-01T00:00:00.000000,5684979000.0,0.0,0.0,0.0,0.0,0.0,68.258804,154.72658,154.72658,2712226.0,856132.9,208552.92,856132.9,647579.94,1221141900.0
+0,2020-10-01T00:00:00.000000,5659591700.0,0.0,0.0,0.0,0.0,0.0,66.956535,156.4753,156.4753,2654457.8,837970.5,214351.67,837970.5,623618.9,1246529300.0
+0,2020-11-01T00:00:00.000000,5634260500.0,0.0,0.0,0.0,0.0,0.0,64.80198,155.93571,155.93571,2580463.0,809597.94,207660.8,809597.94,601937.1,1271860400.0
+0,2020-12-01T00:00:00.000000,5610946000.0,0.0,0.0,0.0,0.0,0.0,62.265244,153.91632,153.91632,2505382.5,777148.44,193741.4,777148.44,583407.06,1295174900.0

tests/unit_tests/test_load_forward_model.py

@@ -7,6 +7,7 @@
 
 import numpy as np
 import pytest
+import xarray as xr
 from resdata.summary import Summary
 
 from ert.config import ErtConfig
@@ -111,6 +112,42 @@ def test_load_inconsistent_time_map_summary(caplog):
     assert loaded == 1
 
 
+@pytest.mark.usefixtures("copy_snake_oil_case_storage")
+def test_datetime_2500():
+    """
+    Test that we are able to work with dates past year 2263 in summary files.
+
+    """
+    cwd = os.getcwd()
+
+    # Get rid of GEN_DATA as we are only interested in SUMMARY
+    with fileinput.input("snake_oil.ert", inplace=True) as fin:
+        for line in fin:
+            if line.startswith("GEN_DATA"):
+                continue
+            print(line, end="")
+
+    facade = LibresFacade.from_config_file("snake_oil.ert")
+    storage_path = ErtConfig.from_file("snake_oil.ert").ens_path
+    storage = open_storage(storage_path, mode="w")
+    ensemble = storage.get_ensemble_by_name("default_0")
+
+    # Create a result that is incompatible with the refcase
+    run_path = Path("storage") / "snake_oil" / "runpath" / "realization-0" / "iter-0"
+    os.chdir(run_path)
+    ecl_sum = run_simulator(100, datetime(2500, 1, 1))
+    ecl_sum.fwrite()
+    os.chdir(cwd)
+
+    realizations = [False] * facade.get_ensemble_size()
+    realizations[0] = True
+    facade.load_from_forward_model(ensemble, realizations, 0)
+
+    dataset = ensemble.load_responses("summary", tuple([0]))
+    assert dataset.coords["time"].data.dtype == np.dtype("object")
+    assert dataset.coords["time"].values[0] == "2500-01-10T00:00:00.000000"
+
+
 @pytest.mark.usefixtures("copy_snake_oil_case_storage")
 def test_load_forward_model(snake_oil_default_storage):
     """