Group observations by response (WIP)

src/ert/analysis/_es_update.py

@@ -224,24 +224,35 @@ def _get_obs_and_measure_data(
                 f"Observation: {observation} attached to response: {group}"
             ) from e
 
-        df = filtered_response.to_dataframe().reset_index()
-
-        observation_keys.append(df["name"].to_list())
-        observation_values.append(df["observations"].to_list())
-        observation_errors.append(df["std"].to_list())
+        observation_keys.extend(
+            observation.to_dataframe()
+            .reset_index()["name"]
+            .to_numpy()
+            .reshape((len(filtered_response.name), -1))
+            .tolist()
+        )
+        observation_values.extend(
+            observation["observations"]
+            .to_dataframe()
+            .to_numpy()
+            .reshape((1, -1))
+            .tolist()
+        )
+        observation_errors.extend(
+            observation["std"].to_dataframe().to_numpy().reshape((1, -1)).tolist()
+        )
 
-        measured_data.append(
+        measured_data.extend(
             filtered_response["values"]
             .transpose(..., "realization")
-            .values.reshape((-1, len(filtered_response.realization)))
+            .values.reshape(
+                (len(filtered_response.name), -1, len(filtered_response.realization))
+            )
         )
     ensemble.load_responses.cache_clear()
-    source_fs.load_responses.cache_clear()
 
-    # Measured_data, an array of 3 dimensions
-    # Outer dimension: One array per observation
-    # Mid dimension is ??? Sometimes length 1, sometimes nreals?
-    # Inner dimension: value is "values", index is realization
+    # Measured_data, an array of nd arrays with shape (1, nreals)
+    # Each inner array has 1 dimension containing obs key, and nreals "values"
     return (
         np.concatenate(measured_data, axis=0),
         np.concatenate(observation_values),

src/ert/dark_storage/common.py

@@ -148,17 +148,20 @@ def data_for_key(
 
 def get_all_observations(experiment: Experiment) -> List[Dict[str, Any]]:
     observations = []
-    for key, dataset in experiment.observations.items():
-        observation = {
-            "name": key,
-            "values": list(dataset["observations"].values.flatten()),
-            "errors": list(dataset["std"].values.flatten()),
-        }
-        if "time" in dataset.coords:
-            observation["x_axis"] = _prepare_x_axis(dataset["time"].values.flatten())  # type: ignore
-        else:
-            observation["x_axis"] = _prepare_x_axis(dataset["index"].values.flatten())  # type: ignore
-        observations.append(observation)
+    for response_key, dataset in experiment.observations.items():
+        x_coord_key = "time" if "time" in dataset.coords else "index"
+
+        for obs_name in dataset["name"].values.flatten():
+            ds_for_name = dataset.sel(name=obs_name)
+            df_for_name = ds_for_name.reset_index()
+            observations.append({
+                "name": obs_name,
+                "values": df_for_name["values"].to_list(),
+                "errors": df_for_name["std"].to_list(),
+                "x_axis": _prepare_x_axis(df_for_name[x_coord_key].to_list())
+            })
+
+        observations.extend(dataset)
 
     observations.sort(key=lambda x: x["x_axis"])  # type: ignore
     return observations
@@ -171,15 +174,16 @@ def get_observations_for_obs_keys(
     experiment_observations = ensemble.experiment.observations
     for key in observation_keys:
         dataset = experiment_observations[key]
+        df = dataset.to_dataframe().reset_index()
         observation = {
             "name": key,
-            "values": list(dataset["observations"].values.flatten()),
-            "errors": list(dataset["std"].values.flatten()),
+            "values": list(df["observations"].to_list()),
+            "errors": list(df["std"].to_list()),
         }
         if "time" in dataset.coords:
-            observation["x_axis"] = _prepare_x_axis(dataset["time"].values.flatten())  # type: ignore
+            observation["x_axis"] = _prepare_x_axis(df["time"].to_list()) # type: ignore
         else:
-            observation["x_axis"] = _prepare_x_axis(dataset["index"].values.flatten())  # type: ignore
+            observation["x_axis"] = _prepare_x_axis(df["index"].to_list()) # type: ignore
         observations.append(observation)
 
     observations.sort(key=lambda x: x["x_axis"])  # type: ignore

src/ert/dark_storage/endpoints/records.py

@@ -113,12 +113,12 @@ def get_ensemble_responses(
     response_names_with_observations = set()
     for dataset in ensemble.experiment.observations.values():
         if dataset.attrs["response"] == "summary" and "name" in dataset.coords:
-            response_name = dataset.name.values.flatten()[0]
-            response_names_with_observations.add(response_name)
+            summary_kw_names = dataset.name.values.flatten()
+            response_names_with_observations = response_names_with_observations.union(set(summary_kw_names))
         else:
             response_name = dataset.attrs["response"]
             if "report_step" in dataset.coords:
-                report_step = dataset.report_step.values.flatten()[0]
+                report_step = dataset.report_step.values.flatten()
             response_names_with_observations.add(response_name + "@" + str(report_step))
 
     for name in ensemble.get_summary_keyset():