[FSTORE-1371] Move to pyproject, add badge for supported python version (#1307)

* Add python version and ruff badge

* Update README.md

* Add info to pyproject.toml

* Move to pyproject.toml

* Update ruff, minor adjustments toml

* Switch email

* Add minor Readme changes, further needed

* Minor readme update for app connection

* Correct typo

* Remove comment about not existing setup section

* Fix missing new_line

* Apply ruff format manually to non-changed files

* Update ruff

* Remove manifest.in

* remove ruff pin version

* Include actual package

* Remove console.log

* Add upper limit to the python version

* Update README.md

Co-authored-by: Robin Andersson <[email protected]>

---------

Co-authored-by: Robin Andersson <[email protected]>

Author: vatj

.github/workflows/python-lint.yml

@@ -26,7 +26,7 @@ jobs:
               - 'python/tests/**/*.py'
 
       - name: install deps
-        run: pip install ruff==0.3.5
+        run: pip install ruff==0.4.2
 
       - name: ruff on python files
         if: steps.get-changed-files.outputs.src_any_changed == 'true'

README.md

@@ -9,6 +9,10 @@
     src="https://img.shields.io/badge/docs-HSFS-orange"
     alt="Hopsworks Feature Store Documentation"
   /></a>
+  <a><img
+    src="https://img.shields.io/badge/python-3.8+-blue"
+    alt="python"
+  /></a>
   <a href="https://pypi.org/project/hsfs/"><img
     src="https://img.shields.io/pypi/v/hsfs?color=blue"
     alt="PyPiStatus"
@@ -21,9 +25,9 @@
     src="https://pepy.tech/badge/hsfs/month"
     alt="Downloads"
   /></a>
-  <a href="https://github.com/psf/black"><img
-    src="https://img.shields.io/badge/code%20style-black-000000.svg"
-    alt="CodeStyle"
+  <a href=https://github.com/astral-sh/ruff><img
+    src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json"
+    alt="Ruff"
   /></a>
   <a><img
     src="https://img.shields.io/pypi/l/hsfs?color=green"
@@ -41,19 +45,44 @@ The library is environment independent and can be used in two modes:
 
 The library automatically configures itself based on the environment it is run.
 However, to connect from an external environment such as Databricks or AWS Sagemaker,
-additional connection information, such as host and port, is required. For more information about the setup from external environments, see the setup section.
+additional connection information, such as host and port, is required. For more information checkout the [Hopsworks documentation](https://docs.hopsworks.ai/latest/).
 
 ## Getting Started On Hopsworks
 
-Instantiate a connection and get the project feature store handler
+Get started easily by registering an account on [Hopsworks Serverless](https://app.hopsworks.ai/). Create your project and a [new Api key](https://docs.hopsworks.ai/latest/user_guides/projects/api_key/create_api_key/). In a new python environment with Python 3.8 or higher, install the [client library](https://docs.hopsworks.ai/latest/user_guides/client_installation/) using pip:
+
+```bash
+# Get all Hopsworks SDKs: Feature Store, Model Serving and Platform SDK
+pip install hopsworks
+# or minimum install with the Feature Store SDK
+pip install hsfs[python]
+# if using zsh don't forget the quotes
+pip install 'hsfs[python]'
+```
+
+You can start a notebook and instantiate a connection and get the project feature store handler.
+
+```python
+import hopsworks
+
+project = hopsworks.login() # you will be prompted for your api key
+fs = project.get_feature_store()
+```
+
+or using `hsfs` directly:
+
 ```python
 import hsfs
 
-connection = hsfs.connection()
+connection = hsfs.connection(
+    host="c.app.hopsworks.ai", #
+    project="your-project",
+    api_key_value="your-api-key",
+)
 fs = connection.get_feature_store()
 ```
 
-Create a new feature group
+Create a new feature group to start inserting feature values.
 ```python
 fg = fs.create_feature_group("rain",
                         version=1,
@@ -135,7 +164,7 @@ You can find more examples on how to use the library in our [hops-examples](http
 
 ## Usage
 
-Usage data is collected for improving quality of the library. It is turned on by default if the backend 
+Usage data is collected for improving quality of the library. It is turned on by default if the backend
 is "c.app.hopsworks.ai". To turn it off, use one of the following way:
 ```python
 # use environment variable

docs/css/dropdown.css

@@ -12,10 +12,7 @@
 .md-tabs {
   overflow: inherit;
 }
-/*
-.md-header {
-  z-index: 1000;
-} */
+
 
 /* The container <div> - needed to position the dropdown content */
 .dropdown {
@@ -55,4 +52,4 @@
 }
 
 /* Change the background color of the dropdown button when the dropdown content is shown */
-.dropdown:hover .dropbtn {}
+.dropdown:hover .dropbtn {}

docs/js/inject-api-links.js

@@ -9,13 +9,11 @@ window.addEventListener("DOMContentLoaded", function () {
         document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/";
     } else { // on docs.hopsworks.api/feature-store-api/3.0 / docs.hopsworks.api/hopsworks-api/3.0 / docs.hopsworks.api/machine-learning-api/3.0
         if (latestRegex.test(windowPathNameSplits[2]) || latestRegex.test(windowPathNameSplits[1])) {
-          console.log("latest version");
           var majorVersion = "latest";
         } else {
 
           var apiVersion = windowPathNameSplits[2];
           var majorVersion = apiVersion.match(majorVersionRegex)[0];
-          console.log("specific version", majorVersion);
         }
         // Version main navigation
         document.getElementsByClassName("md-tabs__link")[0].href = "https://docs.hopsworks.ai/" + majorVersion;

docs/js/version-select.js

@@ -29,11 +29,9 @@ window.addEventListener("DOMContentLoaded", function() {
       return i.version === CURRENT_VERSION ||
              i.aliases.includes(CURRENT_VERSION);
     }).version;
-    console.log("Current version: " + realVersion);
     var latestVersion = versions.find(function(i) {
       return i.aliases.includes("latest");
     }).version;
-    console.log("Latest version: " + latestVersion);
     let outdated_banner = document.querySelector('div[data-md-color-scheme="default"][data-md-component="outdated"]');
     if (realVersion !== latestVersion) {
       outdated_banner.removeAttribute("hidden");

mkdocs.yml

@@ -45,6 +45,7 @@ nav:
           - EmbeddingIndex: generated/api/embedding_index_api.md
           - EmbeddingFeature: generated/api/embedding_feature_api.md
           - SimilarityFunctionType: generated/api/similarity_function_type_api.md
+    # Added to allow navigation using the side drawer
     - Hopsworks API: https://docs.hopsworks.ai/hopsworks-api/latest/
     - MLOps API: https://docs.hopsworks.ai/machine-learning-api/latest/
     - Feature Store JavaDoc: https://docs.hopsworks.ai/feature-store-javadoc/latest/

python/.pre-commit-config.yaml

@@ -1,13 +1,8 @@
 exclude: setup.py
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.4.2
     hooks:
       - id: ruff
         args: [--fix]
-      - id: ruff-format
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
+      - id: ruff-format

python/MANIFEST.in

@@ -34,8 +34,10 @@
 
 
 def _is_timeout(exception):
-    return (isinstance(exception, urllib3.exceptions.ReadTimeoutError)
-            or isinstance(exception, ConnectionTimeout))
+    return isinstance(exception, urllib3.exceptions.ReadTimeoutError) or isinstance(
+        exception, ConnectionTimeout
+    )
+
 
 def _handle_opensearch_exception(func):
     @wraps(func)
@@ -44,15 +46,18 @@ def error_handler_wrapper(*args, **kw):
             return func(*args, **kw)
         except Exception as e:
             if _is_timeout(e):
-                raise FeatureStoreException(OpenSearchClientSingleton.TIMEOUT_ERROR_MSG) from e
+                raise FeatureStoreException(
+                    OpenSearchClientSingleton.TIMEOUT_ERROR_MSG
+                ) from e
             else:
                 raise e
 
     return error_handler_wrapper
 
+
 class OpensearchRequestOption:
     DEFAULT_OPTION_MAP = {
-       "timeout": 30,
+        "timeout": 30,
     }
 
     @classmethod
@@ -122,18 +127,24 @@ def _refresh_opensearch_connection(self):
     @_handle_opensearch_exception
     def search(self, index=None, body=None, options=None):
         try:
-            return self._opensearch_client.search(body=body, index=index, params=OpensearchRequestOption.get_options(options))
+            return self._opensearch_client.search(
+                body=body,
+                index=index,
+                params=OpensearchRequestOption.get_options(options),
+            )
         except (ConnectionError, AuthenticationException):
             # OpenSearchConnectionError occurs when connection is closed.
             # OpenSearchAuthenticationException occurs when jwt is expired
             self._refresh_opensearch_connection()
-            return self._opensearch_client.search(body=body, index=index,
-                                                  params=OpensearchRequestOption.get_options(options))
+            return self._opensearch_client.search(
+                body=body,
+                index=index,
+                params=OpensearchRequestOption.get_options(options),
+            )
         except RequestError as e:
             caused_by = e.info.get("error") and e.info["error"].get("caused_by")
             if caused_by and caused_by["type"] == "illegal_argument_exception":
-                raise self._create_vector_database_exception(
-                    caused_by["reason"]) from e
+                raise self._create_vector_database_exception(caused_by["reason"]) from e
             raise VectorDatabaseException(
                 VectorDatabaseException.OTHERS,
                 f"Error in Opensearch request: {e}",
@@ -147,9 +158,10 @@ def search(self, index=None, body=None, options=None):
     )
     @_handle_opensearch_exception
     def count(self, index, body=None, options=None):
-        result = self._opensearch_client.count(index=index, body=body,
-                                               params=OpensearchRequestOption.get_options(options))
-        return result['count']
+        result = self._opensearch_client.count(
+            index=index, body=body, params=OpensearchRequestOption.get_options(options)
+        )
+        return result["count"]
 
     def close(self):
         if self._opensearch_client:
@@ -166,9 +178,7 @@ def _create_vector_database_exception(self, message):
                     f"Illegal argument in vector database request: "
                     f"Requested k is too large, it needs to be less than {k}."
                 )
-                info = {
-                    VectorDatabaseException.REQUESTED_K_TOO_LARGE_INFO_K: int(
-                        k)}
+                info = {VectorDatabaseException.REQUESTED_K_TOO_LARGE_INFO_K: int(k)}
             else:
                 reason = VectorDatabaseException.REQUESTED_K_TOO_LARGE
                 message = "Illegal argument in vector database request: Requested k is too large."

python/hsfs/core/opensearch.py

@@ -342,9 +342,7 @@ def count(self, options: map = None):
             FeaturestoreException: If an error occurs during the count operation.
         """
         if self._vector_db_client is None:
-            self._vector_db_client = VectorDbClient(
-                self._feature_group.select_all()
-            )
+            self._vector_db_client = VectorDbClient(self._feature_group.select_all())
         return self._vector_db_client.count(self.feature_group, options=options)
 
     @classmethod

python/hsfs/embedding.py

@@ -1,3 +1,118 @@
+[project]
+name = "hsfs"
+dynamic = ["version"]
+requires-python = ">=3.8,<=3.12"
+readme = "README.md"
+description = "HSFS Python SDK to interact with Hopsworks Feature Store"
+keywords = [
+    "Hopsworks",
+    "Feature Store",
+    "hsfs",
+    "Spark",
+    "Machine Learning",
+    "MLOps",
+    "DataOps",
+]
+authors = [{ name = "Hopsworks AB", email = "[email protected]" }]
+license = { text = "Apache-2.0" }
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Utilities",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+]
+
+dependencies = [
+    "pyhumps==1.6.1",
+    "requests",
+    "furl",
+    "boto3",
+    "pandas<2.2.0",
+    "numpy<2",
+    "pyjks",
+    "mock",
+    "avro==1.11.3",
+    "sqlalchemy",
+    "PyMySQL[rsa]",
+    "great_expectations==0.18.12",
+    "tzlocal",
+    "fsspec",
+    "retrying",
+    "aiomysql[sa] @ git+https://[email protected]/logicalclocks/aiomysql",
+    "polars>=0.20.18,<=0.21.0",
+    "opensearch-py>=1.1.0,<=2.4.2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest==7.4.4",
+    "pytest-mock==3.12.0",
+    "ruff",
+    "pyspark==3.1.1",
+    "moto[s3]==5.0.0",
+    "typeguard==4.2.1",
+]
+dev-pandas1 = [
+    "pytest==7.4.4",
+    "pytest-mock==3.12.0",
+    "ruff",
+    "pyspark==3.1.1",
+    "moto[s3]==5.0.0",
+    "pandas<=1.5.3",
+    "sqlalchemy<=1.4.48",
+]
+docs = [
+    "mkdocs==1.5.3",
+    "mkdocs-material==9.5.17",
+    "mike==2.0.0",
+    "sphinx==7.2.6",
+    "keras_autodoc @ git+https://[email protected]/logicalclocks/keras-autodoc",
+    "markdown-include==0.8.1",
+    "mkdocs-jupyter==0.24.3",
+    "markdown==3.6",
+    "pymdown-extensions==10.7.1",
+    "mkdocs-macros-plugin==1.0.4",
+    "mkdocs-minify-plugin>=0.2.0",
+]
+hive = [
+    "pyhopshive[thrift]",
+    "pyarrow>=10.0",
+    "confluent-kafka<=2.3.0",
+    "fastavro>=1.4.11,<=1.8.4",
+]
+python = [
+    "pyhopshive[thrift]",
+    "pyarrow>=10.0",
+    "confluent-kafka<=2.3.0",
+    "fastavro>=1.4.11,<=1.8.4",
+    "tqdm",
+]
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+exclude = ["tests*"]
+include = ["../Readme.md", "../LICENSE", "hsfs"]
+
+[tool.setuptools.dynamic]
+version = { attr = "hsfs.version.__version__" }
+
+[project.urls]
+Documentation = "https://docs.hopsworks.ai/latest"
+Repository = "https://github.com/logicalclocks/feature-store-api"
+Homepage = "https://www.hopsworks.ai"
+Community = "https://community.hopsworks.ai"
+
+
 [tool.ruff]
 # Exclude a variety of commonly ignored directories.
 exclude = [
@@ -39,14 +154,15 @@ target-version = "py38"
 
 [tool.ruff.lint]
 # 1. Enable flake8-bugbear (`B`) rules, in addition to the defaults.
-select = ["E4", "E7", "E9", "F", "B", "I"]#, "ANN"]
+select = ["E4", "E7", "E9", "F", "B", "I", "W"] #, "ANN"]
 ignore = [
     "B905",   # zip has no strict kwarg until Python 3.10
     "ANN101", # Missing type annotation for self in method
     "ANN102", # Missing type annotation for cls in classmethod
     "ANN003", # Missing type annotation for **kwarg in function
     "ANN002", # Missing type annotation for *args in function
     "ANN401", # Allow Any in type annotations
+    "W505",   # Doc line too long
 ]
 
 # Allow fix for all enabled rules (when `--fix`) is provided.