Rewrite for named tuples (#852)

## Title

Refactor mlos_core APIs to encapsulate related data fields.

## Description

Refactors the mlos_core Optimizer APIs to accept new data types
`Observation`, `Observations` and return `Suggestion`, instead of a mess
of `Tuple[DataFrame, DataFrame, Optional[DataFrame],
Optional[DataFrame]]` that must be named and checked everywhere.

Additionally, this makes it more explicit that `_register` is a bulk
operation that is not actually supported currently by the underlying
optimizers, though leaves notes on how we can do that in the future.

## Type of Change

- Refactor

---

## Testing

Usual CI plus some new unit tests for new data type operations.

---

## Additional Notes

A more significant rewrite of named tuple support inside mlos_core.
This is based on comments in #811 as well as conversations with @bpkroth

---

---------

Co-authored-by: Brian Kroth <[email protected]>
Co-authored-by: Brian Kroth <[email protected]>

Author: jsfreischuetz

Diff for: .cspell.json

@@ -21,6 +21,7 @@
         "discretization",
         "discretize",
         "drivername",
+        "dropna",
         "dstpath",
         "dtype",
         "duckdb",

Diff for: mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -21,6 +21,7 @@
 from mlos_bench.services.base_service import Service
 from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_core.data_classes import Observations
 from mlos_core.optimizers import (
     DEFAULT_OPTIMIZER_TYPE,
     BaseOptimizer,
@@ -128,7 +129,7 @@ def bulk_register(
 
         # TODO: Specify (in the config) which metrics to pass to the optimizer.
         # Issue: https://github.com/microsoft/MLOS/issues/745
-        self._opt.register(configs=df_configs, scores=df_scores)
+        self._opt.register(observations=Observations(configs=df_configs, scores=df_scores))
 
         if _LOG.isEnabledFor(logging.DEBUG):
             (score, _) = self.get_best_observation()
@@ -198,10 +199,10 @@ def suggest(self) -> TunableGroups:
         tunables = super().suggest()
         if self._start_with_defaults:
             _LOG.info("Use default values for the first trial")
-        df_config, _metadata = self._opt.suggest(defaults=self._start_with_defaults)
+        suggestion = self._opt.suggest(defaults=self._start_with_defaults)
         self._start_with_defaults = False
-        _LOG.info("Iteration %d :: Suggest:\n%s", self._iter, df_config)
-        return tunables.assign(configspace_data_to_tunable_values(df_config.loc[0].to_dict()))
+        _LOG.info("Iteration %d :: Suggest:\n%s", self._iter, suggestion.config)
+        return tunables.assign(configspace_data_to_tunable_values(suggestion.config.to_dict()))
 
     def register(
         self,
@@ -221,18 +222,20 @@ def register(
             # TODO: Specify (in the config) which metrics to pass to the optimizer.
             # Issue: https://github.com/microsoft/MLOS/issues/745
             self._opt.register(
-                configs=df_config,
-                scores=pd.DataFrame([registered_score], dtype=float),
+                observations=Observations(
+                    configs=df_config,
+                    scores=pd.DataFrame([registered_score], dtype=float),
+                )
             )
         return registered_score
 
     def get_best_observation(
         self,
     ) -> Union[Tuple[Dict[str, float], TunableGroups], Tuple[None, None]]:
-        (df_config, df_score, _df_context) = self._opt.get_best_observations()
-        if len(df_config) == 0:
+        best_observations = self._opt.get_best_observations()
+        if len(best_observations) == 0:
             return (None, None)
-        params = configspace_data_to_tunable_values(df_config.iloc[0].to_dict())
-        scores = self._adjust_signs_df(df_score).iloc[0].to_dict()
+        params = configspace_data_to_tunable_values(best_observations.configs.iloc[0].to_dict())
+        scores = self._adjust_signs_df(best_observations.scores).iloc[0].to_dict()
         _LOG.debug("Best observation: %s score: %s", params, scores)
         return (scores, self._tunables.copy().assign(params))

Diff for: mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py

@@ -16,8 +16,9 @@
 from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer
 from mlos_bench.optimizers.mock_optimizer import MockOptimizer
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_core.data_classes import Suggestion
 from mlos_core.optimizers.bayesian_optimizers.smac_optimizer import SmacOptimizer
-from mlos_core.util import config_to_dataframe
+from mlos_core.util import config_to_series
 
 # For debugging purposes output some warnings which are captured with failed tests.
 DEBUG = True
@@ -40,10 +41,13 @@ def _optimize(env: Environment, opt: Optimizer) -> Tuple[float, TunableGroups]:
             # pylint: disable=protected-access
             if isinstance(opt, MlosCoreOptimizer) and isinstance(opt._opt, SmacOptimizer):
                 config = tunable_values_to_configuration(tunables)
-                config_df = config_to_dataframe(config)
+                config_series = config_to_series(config)
                 logger("config: %s", str(config))
                 try:
-                    logger("prediction: %s", opt._opt.surrogate_predict(configs=config_df))
+                    logger(
+                        "prediction: %s",
+                        opt._opt.surrogate_predict(suggestion=Suggestion(config=config_series)),
+                    )
                 except RuntimeError:
                     pass
 

Diff for: mlos_core/mlos_core/__init__.py

@@ -62,39 +62,38 @@
 ...     space_adapter_kwargs=space_adpaters_kwargs,
 ... )
 >>> # Get a new configuration suggestion.
->>> (config_df, _metadata_df) = opt.suggest()
+>>> suggestion = opt.suggest()
 >>> # Examine the suggested configuration.
->>> assert len(config_df) == 1
->>> config_df.iloc[0]
+>>> assert len(suggestion.config) == 1
+>>> suggestion.config
 x    3
-Name: 0, dtype: int64
+dtype: object
 >>> # Register the configuration and its corresponding target value
 >>> score = 42 # a made up score
->>> scores_df = pandas.DataFrame({"y": [score]})
->>> opt.register(configs=config_df, scores=scores_df)
+>>> scores_sr = pandas.Series({"y": score})
+>>> opt.register(suggestion.complete(scores_sr))
 >>> # Get a new configuration suggestion.
->>> (config_df, _metadata_df) = opt.suggest()
->>> config_df.iloc[0]
+>>> suggestion = opt.suggest()
+>>> suggestion.config
 x    10
-Name: 0, dtype: int64
+dtype: object
 >>> score = 7 # a better made up score
 >>> # Optimizers minimize by convention, so a lower score is better
 >>> # You can use a negative score to maximize values instead
 >>> #
->>> # Convert it to a DataFrame again
->>> scores_df = pandas.DataFrame({"y": [score]})
->>> opt.register(configs=config_df, scores=scores_df)
+>>> # Convert it to a Series again
+>>> scores_sr = pandas.Series({"y": score})
+>>> opt.register(suggestion.complete(scores_sr))
 >>> # Get the best observations.
->>> (configs_df, scores_df, _contexts_df) = opt.get_best_observations()
+>>> observations = opt.get_best_observations()
 >>> # The default is to only return one
->>> assert len(configs_df) == 1
->>> assert len(scores_df) == 1
->>> configs_df.iloc[0]
-x    10
-Name: 1, dtype: int64
->>> scores_df.iloc[0]
-y    7
-Name: 1, dtype: int64
+>>> assert len(observations) == 1
+>>> observations.configs
+    x
+0  10
+>>> observations.scores
+   y
+0  7
 
 Notes
 -----