Adult dataset demo (#837)

* Adult dataset demo

* fix import

* Made more positive test panels

---------

Co-authored-by: Emeli Dral <[email protected]>

src/evidently/cli/ui.py

@@ -15,7 +15,7 @@ def ui(
     demo_projects: str = Option(
         "",
         "--demo-projects",
-        help="Comma-separated list of demo projects to generate. Possible values: [all|bikes|reviews]",
+        help="Comma-separated list of demo projects to generate. Possible values: [all|bikes|reviews|adult]",
     ),
     secret: Optional[str] = Option(None, help="Secret for writing operations"),
 ):

src/evidently/ui/demo_projects/__init__.py

@@ -1,7 +1,12 @@
 from typing import Dict
 
+from .adult import adult_demo_project
 from .base import DemoProject
 from .bikes import bikes_demo_project
 from .reviews import reviews_demo_project
 
-DEMO_PROJECTS: Dict[str, DemoProject] = {"bikes": bikes_demo_project, "reviews": reviews_demo_project}
+DEMO_PROJECTS: Dict[str, DemoProject] = {
+    "bikes": bikes_demo_project,
+    "reviews": reviews_demo_project,
+    "adult": adult_demo_project,
+}

src/evidently/ui/demo_projects/adult.py

@@ -0,0 +1,108 @@
+from datetime import datetime
+from datetime import timedelta
+
+from sklearn import datasets
+
+from evidently import ColumnMapping
+from evidently.renderers.html_widgets import WidgetSize
+from evidently.test_preset import DataDriftTestPreset
+from evidently.test_suite import TestSuite
+from evidently.ui.dashboards import DashboardPanelTestSuite
+from evidently.ui.dashboards import ReportFilter
+from evidently.ui.dashboards import TestFilter
+from evidently.ui.dashboards import TestSuitePanelType
+from evidently.ui.demo_projects.base import DemoProject
+from evidently.ui.workspace import WorkspaceBase
+
+
+def create_data():
+    adult_data = datasets.fetch_openml(name="adult", version=2, as_frame="auto")
+    adult = adult_data.frame
+
+    reference = adult[~adult.education.isin(["Some-college", "HS-grad", "Bachelors"])]
+    current = adult[adult.education.isin(["Some-college", "HS-grad", "Bachelors"])]
+    column_mapping = ColumnMapping()
+    return current, reference, column_mapping
+
+
+def create_test_suite(i: int, data):
+    current, reference, column_mapping = data
+    ts = TestSuite(
+        tests=[
+            DataDriftTestPreset(),
+        ],
+        timestamp=datetime(2023, 1, 29) + timedelta(days=i + 1),
+    )
+    ts.metadata["batch_size"] = "daily"
+
+    ts.run(
+        reference_data=reference,
+        current_data=current.iloc[1000 * i : 1000 * (i + 1), :],
+        column_mapping=column_mapping,
+    )
+
+    return ts
+
+
+def create_project(workspace: WorkspaceBase, name: str):
+    project = workspace.create_project(name)
+    project.description = "A toy demo project using Adult dataset. Showcases TestSuite panels"
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="Column Drift tests for key features: aggregated",
+            test_filters=[
+                TestFilter(test_id="TestColumnDrift", test_args={"column_name.name": "hours-per-week"}),
+                TestFilter(test_id="TestColumnDrift", test_args={"column_name.name": "capital-gain"}),
+            ],
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF,
+            time_agg="1D",
+        )
+    )
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="All tests: aggregated",
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF,
+            time_agg="1D",
+        )
+    )
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="Column Drift tests for key features: detailed",
+            test_filters=[
+                TestFilter(test_id="TestColumnDrift", test_args={"column_name.name": "hours-per-week"}),
+                TestFilter(test_id="TestColumnDrift", test_args={"column_name.name": "capital-gain"}),
+            ],
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF,
+            panel_type=TestSuitePanelType.DETAILED,
+            time_agg="1D",
+        )
+    )
+    project.dashboard.add_panel(
+        DashboardPanelTestSuite(
+            title="All tests: detailed",
+            filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),
+            size=WidgetSize.HALF,
+            panel_type=TestSuitePanelType.DETAILED,
+            time_agg="1D",
+        )
+    )
+
+    project.save()
+    return project
+
+
+adult_demo_project = DemoProject(
+    name="Demo project - Adult",
+    create_data=create_data,
+    create_report=None,
+    create_project=create_project,
+    create_test_suite=create_test_suite,
+    count=19,
+)
+
+if __name__ == "__main__":
+    # create_demo_project("http://localhost:8080")
+    adult_demo_project.create("workspace")

src/evidently/ui/models.py

@@ -69,11 +69,15 @@ def from_project_with_time_range(
         timestamp_start: Optional[datetime.datetime] = None,
         timestamp_end: Optional[datetime.datetime] = None,
     ):
-
-        time_range = dict(
-            min_timestamp=min(r.timestamp for r in project.reports.values()),
-            max_timestamp=max(r.timestamp for r in project.reports.values()),
-        )
+        time_range: Dict[str, Optional[datetime.datetime]]
+        reports = project.reports_and_test_suites
+        if len(reports) == 0:
+            time_range = {"min_timestamp": None, "max_timestamp": None}
+        else:
+            time_range = dict(
+                min_timestamp=min(r.timestamp for r in reports.values()),
+                max_timestamp=max(r.timestamp for r in reports.values()),
+            )
 
         info = project.build_dashboard_info(timestamp_start=timestamp_start, timestamp_end=timestamp_end)
 

tests/cli/test_ui.py

@@ -0,0 +1,18 @@
+import inspect
+
+import pytest
+
+from evidently.cli import app
+from evidently.ui.demo_projects import DEMO_PROJECTS
+
+
+@pytest.fixture()
+def ui_command():
+    command = [c for c in app.registered_commands if c.name == "ui"][0]
+    argspec = inspect.getfullargspec(command.callback)
+    return dict(zip(argspec.annotations.keys(), argspec.defaults))
+
+
+@pytest.mark.parametrize("demo_project", list(DEMO_PROJECTS.keys()))
+def test_all_demo_projects_in_help(demo_project, ui_command):
+    assert demo_project in ui_command["demo_projects"].help

tests/ui/test_demo_project.py

@@ -5,7 +5,7 @@
 
 
 @pytest.mark.parametrize("demo_project", list(DEMO_PROJECTS.keys()))
-def test_create_demo_proejct(demo_project, tmp_path):
+def test_create_demo_project(demo_project, tmp_path):
     dp = DEMO_PROJECTS[demo_project]
     dp.create(str(tmp_path))