Add integration tests for third-party data science libraries (#251)

* Add integration tests for third-party data science libraries

This is another smoke test to detect breakages in file system APIs.
Adds pandas, polars, and duckdb to the test requirements.

* Add `use_pyarrow=True` for polars Parquet reads

Without that, parquet reads on the newest versions result in weird
errors.

* Add existence check after the upload in Pandas integration test

This is more robust than just checking if the upload breaks.

* Add polars test docstring

Author: nicholasjng

README.md

@@ -86,7 +86,7 @@ print(data.head())
 # Polars -- see https://pola-rs.github.io/polars/user-guide/io/cloud-storage/
 import polars as pl
 
-data = pl.read_parquet("lakefs://quickstart/main/lakes.parquet")
+data = pl.read_parquet("lakefs://quickstart/main/lakes.parquet", use_pyarrow=True)
 print(data.head())
 
 

pyproject.toml

@@ -50,6 +50,10 @@ dev = [
     "pytest>=7.4.0",
     "pytest-cov>=4.1.0",
     "pydoclint",
+    # for integration tests.
+    "pandas[parquet]",
+    "polars",
+    "duckdb",
 ]
 docs = [
     "mkdocs",

requirements-dev.txt

@@ -11,26 +11,33 @@ click==8.1.7
 coverage[toml]==7.4.0
 distlib==0.3.8
 docstring-parser-fork==0.0.5
+duckdb==0.9.2
 filelock==3.13.1
 fsspec==2023.12.2
 identify==2.5.33
 iniconfig==2.0.0
 lakefs==0.2.0
-lakefs-sdk==1.5.0
+lakefs-sdk==1.7.0
 nodeenv==1.8.0
+numpy==1.26.3
 packaging==23.2
+pandas[parquet]==2.1.4
 platformdirs==4.1.0
 pluggy==1.3.0
+polars==0.20.3
 pre-commit==3.6.0
+pyarrow==14.0.2
 pydantic==1.10.13
 pydoclint==0.3.8
 pyproject-hooks==1.0.0
 pytest==7.4.4
 pytest-cov==4.1.0
 python-dateutil==2.8.2
+pytz==2023.3.post1
 pyyaml==6.0.1
 six==1.16.0
 typing-extensions==4.9.0
+tzdata==2023.4
 urllib3==2.0.7
 virtualenv==20.25.0
 wrapt==1.16.0

requirements-docs.txt

@@ -34,7 +34,7 @@ fqdn==1.5.1
 fsspec==2023.12.2
 ghp-import==2.1.0
 gitdb==4.0.11
-gitpython==3.1.40
+gitpython==3.1.41
 griffe==0.38.1
 h11==0.14.0
 httpcore==1.0.2
@@ -43,7 +43,7 @@ idna==3.6
 importlib-metadata==7.0.1
 importlib-resources==6.1.1
 ipykernel==6.28.0
-ipython==8.19.0
+ipython==8.20.0
 ipywidgets==8.1.1
 isoduration==20.11.0
 jedi==0.19.1
@@ -55,18 +55,18 @@ jsonschema-specifications==2023.12.1
 jupyter==1.0.0
 jupyter-client==8.6.0
 jupyter-console==6.6.3
-jupyter-core==5.6.1
+jupyter-core==5.7.1
 jupyter-events==0.9.0
 jupyter-lsp==2.2.1
-jupyter-server==2.12.1
+jupyter-server==2.12.3
 jupyter-server-terminals==0.5.1
 jupyterlab==4.0.10
 jupyterlab-pygments==0.3.0
 jupyterlab-server==2.25.2
 jupyterlab-widgets==3.0.9
 jupytext==1.16.0
 lakefs==0.2.0
-lakefs-sdk==1.5.0
+lakefs-sdk==1.7.0
 markdown==3.5.1
 markdown-it-py==3.0.0
 markupsafe==2.1.3
@@ -87,7 +87,7 @@ mkdocs-material==9.5.3
 mkdocs-material-extensions==1.3.1
 mkdocs-section-index==0.3.8
 mkdocstrings[python]==0.24.0
-mkdocstrings-python==1.7.5
+mkdocstrings-python==1.8.0
 mknotebooks==0.8.0
 mypy-extensions==1.0.0
 nbclient==0.9.0
@@ -123,7 +123,7 @@ pyyaml-env-tag==0.1
 pyzmq==25.1.2
 qtconsole==5.5.1
 qtpy==2.4.1
-referencing==0.32.0
+referencing==0.32.1
 regex==2023.12.25
 requests==2.31.0
 rfc3339-validator==0.1.4
@@ -141,13 +141,13 @@ tinycss2==1.2.1
 toml==0.10.2
 tornado==6.4
 traitlets==5.14.1
-types-python-dateutil==2.8.19.14
+types-python-dateutil==2.8.19.20240106
 typing-extensions==4.9.0
 uri-template==1.3.0
 urllib3==2.0.7
 verspec==0.1.0
 watchdog==3.0.0
-wcwidth==0.2.12
+wcwidth==0.2.13
 webcolors==1.13
 webencodings==0.5.1
 websocket-client==1.7.0

tests/smoke_tests/test_integrations.py

@@ -0,0 +1,45 @@
+import duckdb
+import numpy as np
+import pandas as pd
+import polars as pl
+from lakefs.branch import Branch
+from lakefs.repository import Repository
+
+from lakefs_spec.spec import LakeFSFileSystem
+
+storage_options = dict(
+    host="localhost:8000",
+    username="AKIAIOSFOLQUICKSTART",
+    password="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+)
+
+
+def test_pandas_integration(
+    fs: LakeFSFileSystem, repository: Repository, temp_branch: Branch
+) -> None:
+    """Assure the correctness of pandas DataFrame reads and writes, which use `fs.open()`."""
+    df = pd.read_parquet(
+        f"lakefs://{repository.id}/{temp_branch.id}/lakes.parquet", storage_options=storage_options
+    )
+    df["randomcol"] = np.random.randn(len(df.index))
+    df.to_parquet(
+        f"lakefs://{repository.id}/{temp_branch.id}/lakes_new.parquet",
+        storage_options=storage_options,
+    )
+    assert fs.exists(f"lakefs://{repository.id}/{temp_branch.id}/lakes_new.parquet")
+
+
+def test_polars_integration(repository: Repository) -> None:
+    """Test the download and instantiation of polars DataFrames via `fs.open()`."""
+    pl.read_parquet(
+        f"lakefs://{repository.id}/main/lakes.parquet",
+        use_pyarrow=True,
+        storage_options=storage_options,
+    )
+
+
+def test_duckdb_integration(fs: LakeFSFileSystem, repository: Repository) -> None:
+    """Test the correct registration of the lakeFS file system in duckDB."""
+    # see https://duckdb.org/docs/guides/python/filesystems.html
+    duckdb.register_filesystem(fs)
+    duckdb.read_parquet(f"lakefs://{repository.id}/main/lakes.parquet")