[FSTORE-1328] Add list to feature store and richer repr drafts

python/MANIFEST.in

@@ -1,2 +1,3 @@
 include ../README.md
 include ../LICENSE
+include hsfs/helpers/quicktours/feature_store.md

python/hsfs/connection.py

@@ -29,7 +29,7 @@
 )
 from hsfs.core.opensearch import OpenSearchClientSingleton
 from hsfs.decorators import connected, not_connected
-from requests.exceptions import ConnectionError
+from hsfs.helpers import user_messages
 
 
 AWS_DEFAULT_REGION = "default"
@@ -187,7 +187,9 @@ def get_feature_store(
         """
         if not name:
             name = client.get_instance()._project_name
-        return self._feature_store_api.get(util.append_feature_store_suffix(name))
+        fs = self._feature_store_api.get(util.append_feature_store_suffix(name))
+        user_messages.print_connected_to_feature_store_message(fs)
+        return fs
 
     @not_connected
     def connect(self) -> None:
@@ -264,7 +266,6 @@ def connect(self) -> None:
         except (TypeError, ConnectionError):
             self._connected = False
             raise
-        print("Connected. Call `.close()` to terminate connection gracefully.")
 
     def close(self) -> None:
         """Close a connection gracefully.

python/hsfs/core/feature_group_api.py

@@ -15,6 +15,8 @@
 #
 from __future__ import annotations
 
+from typing import List, Union
+
 from hsfs import client, feature_group, feature_group_commit
 from hsfs.core import explicit_provenance, ingestion_job
 
@@ -24,6 +26,42 @@ class FeatureGroupApi:
     ONDEMAND = "ondemand"
     SPINE = "spine"
 
+    def get_all(
+        self,
+        feature_store_id: int,
+        with_features: bool = False,
+    ) -> List[
+        Union[
+            feature_group.FeatureGroup,
+            feature_group.SpineGroup,
+            feature_group.ExternalFeatureGroup,
+        ]
+    ]:
+        """Get a list of feature groups in a feature store.
+
+        :param feature_store_id: feature store id
+        :type feature_store_id: int
+        :param feature_group_type: type of the feature group to return
+        :type feature_group_type: string
+        :return: list of feature group metadata objects
+        :rtype: List[FeatureGroup]
+        """
+        _client = client.get_instance()
+        path_params = [
+            "project",
+            _client._project_id,
+            "featurestores",
+            feature_store_id,
+            "featuregroups",
+        ]
+        query_params = {}
+        if with_features:
+            query_params["expand"] = ["features"]
+
+        return feature_group.FeatureGroup.from_response_json(
+            _client._send_request("GET", path_params, query_params)
+        )
+
     def save(self, feature_group_instance):
         """Save feature group metadata to the feature store.
 

python/hsfs/core/feature_group_engine.py

@@ -15,12 +15,14 @@
 from __future__ import annotations
 
 import warnings
+from typing import List, Union
 
 from hsfs import engine, util
 from hsfs import feature_group as fg
 from hsfs.client import exceptions
 from hsfs.core import delta_engine, feature_group_base_engine, hudi_engine
 from hsfs.core.deltastreamer_jobconf import DeltaStreamerJobConf
+from hsfs.helpers.richer_repr import richer_feature_group
 
 
 class FeatureGroupEngine(feature_group_base_engine.FeatureGroupBaseEngine):
@@ -388,3 +390,65 @@ def save_feature_group_metadata(
                 feature_group_id=feature_group.id,
             )
         )
+
+    def list_feature_groups(
+        self,
+        latest_version_only: bool,
+        online_enabled_only: bool,
+        spine_only: bool,
+        external_only: bool,
+        with_features: bool,
+    ) -> List[fg.FeatureGroup, fg.ExternalFeatureGroup, fg.SpineGroup]:
+        fg_list = self._feature_group_api.get_all(
+            feature_store_id=self._feature_store_id,
+            with_features=with_features,
+        )
+
+        if online_enabled_only:
+            fg_list = [fgroup for fgroup in fg_list if fgroup.online_enabled]
+        if spine_only:
+            fg_list = [
+                fgroup for fgroup in fg_list if isinstance(fgroup, fg.SpineGroup)
+            ]
+        if external_only:
+            fg_list = [
+                fgroup
+                for fgroup in fg_list
+                if isinstance(fgroup, fg.ExternalFeatureGroup)
+            ]
+
+        if latest_version_only:
+            fg_list = [
+                fgroup
+                for fgroup in fg_list
+                if fgroup.version
+                == max(fg1.version for fg1 in fg_list if fg1.name == fgroup.name)
+            ]
+
+        return sorted(fg_list, key=lambda fgroup: fgroup.name)
+
+    def show_info(
+        self,
+        feature_group: Union[fg.FeatureGroup, fg.ExternalFeatureGroup, fg.SpineGroup],
+        show_features: bool = False,
+    ):
+        richer_feature_group.build_and_print_info_fg_table(
+            feature_group, show_features=show_features
+        )
+
+    def show_all(
+        self,
+        latest_version_only: bool = True,
+        show_features: bool = False,
+        show_description: bool = False,
+    ):
+        fgroup_list = self.list_feature_groups(
+            latest_version_only=latest_version_only,
+            online_enabled_only=False,
+            spine_only=False,
+            external_only=False,
+            with_features=show_features,
+        )
+        richer_feature_group.show_rich_table_feature_groups(
+            fgroup_list, show_features=show_features, show_description=show_description
+        )

python/hsfs/core/feature_view_api.py

@@ -15,7 +15,7 @@
 #
 from __future__ import annotations
 
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from hsfs import (
     client,
@@ -55,6 +55,21 @@ def __init__(self, feature_store_id: int) -> None:
             "featureview",
         ]
 
+    def get_all(
+        self, latest_version_only: bool = True, with_features: bool = False
+    ) -> List[Dict[str, Any]]:
+        path = self._base_path
+        query_params = {}
+        if latest_version_only:
+            query_params["filter_by"] = "latest_version"
+        if with_features:
+            query_params["expand"] = ["features"]
+        return self._client._send_request(
+            "GET",
+            path_params=path,
+            query_params={"expand": ["features"]} if with_features else None,
+        )["items"]
+
     def post(
         self, feature_view_obj: feature_view.FeatureView
     ) -> feature_view.FeatureView:

python/hsfs/core/feature_view_engine.py

@@ -17,8 +17,9 @@
 
 import datetime
 import warnings
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
+import humps
 from hsfs import (
     client,
     engine,
@@ -39,6 +40,7 @@
     training_dataset_engine,
     transformation_function_engine,
 )
+from hsfs.helpers.richer_repr import richer_feature_view
 from hsfs.training_dataset_split import TrainingDatasetSplit
 
 
@@ -923,3 +925,45 @@ def _check_if_exists_with_prefix(self, f_name, f_set):
             )
         else:
             return f_name
+
+    def list_feature_views(
+        self, latest_version_only: bool = True, with_features: bool = False
+    ) -> List[Dict[str, Any]]:
+        fv_list = [
+            humps.decamelize(fv_obj)
+            for fv_obj in self._feature_view_api.get_all(
+                latest_version_only=latest_version_only, with_features=with_features
+            )
+        ]
+        if latest_version_only:
+            fv_list = [
+                fview
+                for fview in fv_list
+                if fview["version"]
+                == max(
+                    fv1["version"] for fv1 in fv_list if fv1["name"] == fview["name"]
+                )
+            ]
+
+        return sorted(fv_list, key=lambda fview: fview["name"])
+
+    def show_info(
+        self, feature_view_obj: feature_view.FeatureView, show_features: bool = True
+    ) -> None:
+        richer_feature_view.build_and_print_info_fv_table(
+            feature_view_obj, show_features=show_features
+        )
+
+    def show_all(
+        self,
+        latest_version_only: bool = True,
+        show_features: bool = False,
+        show_description: bool = False,
+    ):
+        fview_dicts = self.list_feature_views(
+            latest_version_only=latest_version_only,
+            with_features=show_features,
+        )
+        richer_feature_view.show_rich_table_feature_views(
+            fview_dicts, show_features, show_description
+        )

python/hsfs/feature_group.py

@@ -3262,6 +3262,9 @@ def _is_time_travel_enabled(self) -> bool:
             and self._time_travel_format.upper() == "HUDI"
         )
 
+    def _show_info(self, show_features: bool = True) -> None:
+        self._feature_group_engine.show_info(self, show_features=show_features)
+
     @property
     def id(self) -> Optional[int]:
         """Feature group id."""

python/hsfs/feature_store.py

@@ -48,6 +48,7 @@
 )
 from hsfs.decorators import typechecked
 from hsfs.embedding import EmbeddingIndex
+from hsfs.helpers import quicktours, verbose
 from hsfs.statistics_config import StatisticsConfig
 from hsfs.transformation_function import TransformationFunction
 
@@ -736,6 +737,10 @@ def get_or_create_feature_group(
                 self.id, name, version, feature_group_api.FeatureGroupApi.CACHED
             )
             feature_group_object.feature_store = self
+            if verbose.is_hsfs_verbose():
+                self._feature_group_engine.show_info(
+                    feature_group_object, show_features=True
+                )
             return feature_group_object
         except exceptions.RestAPIError as e:
             if (
@@ -764,6 +769,10 @@ def get_or_create_feature_group(
                     notification_topic_name=notification_topic_name,
                 )
                 feature_group_object.feature_store = self
+                if verbose.is_hsfs_verbose():
+                    self._feature_group_engine.show_info(
+                        feature_group_object, show_features=False
+                    )
                 return feature_group_object
             else:
                 raise e
@@ -1639,13 +1648,16 @@ def get_or_create_feature_view(
             `FeatureView`: The feature view metadata object.
         """
         try:
-            return self._feature_view_engine.get(name, version)
+            feature_view_object = self._feature_view_engine.get(name, version)
+            if verbose.is_hsfs_verbose():
+                self._feature_view_engine.show_info(feature_view_object)
+            return feature_view_object
         except exceptions.RestAPIError as e:
             if (
                 e.response.json().get("errorCode", "") == 270181
                 and e.response.status_code == 404
             ):
-                return self.create_feature_view(
+                feature_view_object = self.create_feature_view(
                     name=name,
                     query=query,
                     version=version,
@@ -1655,6 +1667,9 @@ def get_or_create_feature_view(
                     training_helper_columns=training_helper_columns or [],
                     transformation_functions=transformation_functions or {},
                 )
+                if verbose.is_hsfs_verbose():
+                    self._feature_view_engine.show_info(feature_view_object)
+                return feature_view_object
             else:
                 raise e
 
@@ -1728,6 +1743,66 @@ def get_feature_views(self, name: str) -> List["feature_view.FeatureView"]:
         """
         return self._feature_view_engine.get(name)
 
+    def show_feature_groups(
+        self,
+        latest_only: bool = True,
+        show_features: bool = False,
+        show_description: bool = False,
+    ) -> None:
+        """Prints a list of all feature groups in the feature store.
+
+        !!! example
+            ```python
+            # get feature store instance
+            fs = ...
+
+            # show all feature groups
+            fs.show_feature_groups()
+            ```
+
+        # Arguments
+            latest_only: If `True` only the latest version of each feature group is shown, defaults to `True`.
+            show_features: If `True` also show the features of the feature groups, defaults to `False`.
+            show_description: If `True` also show the description of the feature groups, defaults to `False`.
+
+        # Returns
+            `None`
+        """
+        self._feature_group_engine.show_all(
+            latest_only, show_features, show_description
+        )
+
+    def show_feature_views(
+        self,
+        latest_only: bool = True,
+        show_features: bool = False,
+        show_description: bool = False,
+    ) -> None:
+        """Prints a list of all feature views in the feature store.
+
+        !!! example
+            ```python
+            # get feature store instance
+            fs = ...
+
+            # show all feature views
+            fs.show_feature_views()
+            ```
+
+        # Arguments
+            latest_only: If `True` only the latest version of each feature view is shown, defaults to `True`.
+            show_features: If `True` also show the features of the feature views, defaults to `False`.
+            show_description: If `True` also show the description of the feature views, defaults to `False`.
+
+        # Returns
+            `None`
+        """
+        self._feature_view_engine.show_all(latest_only, show_features, show_description)
+
+    def quicktour(self) -> None:
+        """Prints a quick tour of the feature store API."""
+        quicktours.rich_print_quicktour("feature_store")
+
     @property
     def id(self) -> int:
         """Id of the feature store."""