fix for lift metrics (#835)

* Fixed lift metrics * Fixed how_to_make_custom_metric_and_test.ipynb to support json render for a custom metric --------- Co-authored-by: Emeli Dral <[email protected]>
evidentlyai · Oct 26, 2023 · 37d754b · 37d754b
1 parent 53bda86
commit 37d754b
Show file tree

Hide file tree

Showing 8 changed files with 357 additions and 272 deletions.
diff --git a/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb b/examples/how_to_questions/how_to_make_custom_metric_and_test.ipynb
diff --git a/examples/sample_notebooks/evidently_metrics.ipynb b/examples/sample_notebooks/evidently_metrics.ipynb
@@ -75,6 +75,8 @@
     "from evidently.metrics import ClassificationRocCurve\n",
     "from evidently.metrics import ClassificationPRCurve\n",
     "from evidently.metrics import ClassificationPRTable\n",
+    "from evidently.metrics import ClassificationLiftCurve\n",
+    "from evidently.metrics import ClassificationLiftTable\n",
     "from evidently.metrics import ClassificationQualityByFeatureTable\n",
     "from evidently.metrics import ClassificationDummyMetric\n",
     "from evidently.metrics import RegressionQualityMetric\n",
@@ -200,8 +202,7 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    },
-    "scrolled": false
+    }
    },
    "outputs": [],
    "source": [
@@ -220,8 +221,7 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    },
-    "scrolled": false
+    }
    },
    "outputs": [],
    "source": [
@@ -235,8 +235,7 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    },
-    "scrolled": false
+    }
    },
    "outputs": [],
    "source": [
@@ -420,7 +419,7 @@
     "pycharm": {
      "name": "#%%\n"
     },
-    "scrolled": false
+    "scrolled": true
    },
    "outputs": [],
    "source": [
@@ -437,9 +436,9 @@
     "    ClassificationRocCurve(),\n",
     "    ClassificationPRCurve(),\n",
     "    ClassificationPRTable(),\n",
+    "    ClassificationLiftCurve(),\n",
+    "    ClassificationLiftTable(),\n",
     "    ClassificationQualityByFeatureTable(columns=['mean area', 'fractal dimension error']),\n",
-    "\n",
-    "    \n",
     "])\n",
     "\n",
     "classification_report.run(reference_data=bcancer_ref, current_data=bcancer_cur)\n",
@@ -463,8 +462,7 @@
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
-    },
-    "scrolled": false
+    }
    },
    "outputs": [],
    "source": [
@@ -635,9 +633,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
 }
diff --git a/src/evidently/metric_results.py b/src/evidently/metric_results.py
@@ -320,6 +320,28 @@ class Config:
 ROCCurve = Dict[Label, ROCCurveData]
 
 
+class LiftCurveData(MetricResult):
+    class Config:
+        dict_include = False
+        tags = {IncludeTags.Render}
+
+    lift: PlotData
+    top: PlotData
+    count: PlotData
+    prob: PlotData
+    tp: PlotData
+    fp: PlotData
+    precision: PlotData
+    recall: PlotData
+    f1_score: PlotData
+    max_lift: PlotData
+    relative_lift: PlotData
+    percent: PlotData
+
+
+LiftCurve = Dict[Label, LiftCurveData]
+
+
 class HistogramData(MetricResult):
     class Config:
         dict_include = False

diff --git a/src/evidently/metrics/__init__.py b/src/evidently/metrics/__init__.py
@@ -9,6 +9,8 @@
 from .classification_performance.classification_dummy_metric import ClassificationDummyMetric
 from .classification_performance.classification_quality_metric import ClassificationQualityMetric
 from .classification_performance.confusion_matrix_metric import ClassificationConfusionMatrix
+from .classification_performance.lift_curve_metric import ClassificationLiftCurve
+from .classification_performance.lift_table_metric import ClassificationLiftTable
 from .classification_performance.pr_curve_metric import ClassificationPRCurve
 from .classification_performance.pr_table_metric import ClassificationPRTable
 from .classification_performance.probability_distribution_metric import ClassificationProbDistribution
@@ -71,6 +73,8 @@
     "ClassificationQualityByClass",
     "ClassificationQualityByFeatureTable",
     "ClassificationRocCurve",
+    "ClassificationLiftCurve",
+    "ClassificationLiftTable",
     "ColumnDriftMetric",
     "ColumnValuePlot",
     "DataDriftTable",

diff --git a/src/evidently/metrics/classification_performance/lift_curve_metric.py b/src/evidently/metrics/classification_performance/lift_curve_metric.py
@@ -4,11 +4,13 @@
 import pandas as pd
 
 from evidently.base_metric import InputData
+from evidently.base_metric import Metric
 from evidently.base_metric import MetricResult
-from evidently.calculations.classification_performance import PredictionData
 from evidently.calculations.classification_performance import calculate_lift_table
 from evidently.calculations.classification_performance import get_prediction_data
-from evidently.metrics.base_metric import Metric
+from evidently.metric_results import LiftCurve
+from evidently.metric_results import LiftCurveData
+from evidently.metric_results import PredictionData
 from evidently.model.widget import BaseWidgetInfo
 from evidently.renderers.base_renderer import MetricRenderer
 from evidently.renderers.base_renderer import default_renderer
@@ -20,8 +22,8 @@
 
 
 class ClassificationLiftCurveResults(MetricResult):
-    current_lift_curve: Optional[dict] = None
-    reference_lift_curve: Optional[dict] = None
+    current_lift_curve: Optional[LiftCurve] = None
+    reference_lift_curve: Optional[LiftCurve] = None
 
 
 class ClassificationLiftCurve(Metric[ClassificationLiftCurveResults]):
@@ -46,7 +48,7 @@ def calculate(self, data: InputData) -> ClassificationLiftCurveResults:
             reference_lift_curve=ref_lift_curve,
         )
 
-    def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData):
+    def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData) -> LiftCurve:
         labels = prediction.labels
         if prediction.prediction_probas is None:
             raise ValueError("Lift Curve can be calculated only " "on binary probabilistic predictions")
@@ -63,22 +65,23 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData):
                     prediction.prediction_probas.iloc[:, 0].tolist(),
                 )
             )
-            lift_table[int(prediction.prediction_probas.columns[0])] = calculate_lift_table(binded)
+            lift_table[prediction.prediction_probas.columns[0]] = calculate_lift_table(binded)
 
-            lift_curve[int(prediction.prediction_probas.columns[0])] = {
-                "lift": [i[8] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "top": [i[0] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "count": [i[1] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "prob": [i[2] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "tp": [i[3] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "fp": [i[4] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "precision": [i[5] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "recall": [i[6] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "f1_score": [i[7] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "max_lift": [i[9] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "relative_lift": [i[10] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                "percent": lift_table[prediction.prediction_probas.columns[0]][0][11],
-            }
+            lift_curve[prediction.prediction_probas.columns[0]] = LiftCurveData(
+                lift=[i[8] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                top=[i[0] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                count=[i[1] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                prob=[i[2] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                tp=[i[3] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                fp=[i[4] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                precision=[i[5] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                recall=[i[6] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                f1_score=[i[7] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                max_lift=[i[9] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                relative_lift=[i[10] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                percent=[i[11] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                # percent = lift_table[prediction.prediction_probas.columns[0]][0][11],
+            )
         else:
             binaraized_target = pd.DataFrame(binaraized_target)
             binaraized_target.columns = labels
@@ -90,32 +93,33 @@ def calculate_metrics(self, target_data: pd.Series, prediction: PredictionData):
                         prediction.prediction_probas[label],
                     )
                 )
-                lift_table[int(label)] = calculate_lift_table(binded)
+                lift_table[label] = calculate_lift_table(binded)
 
             for label in labels:
-
-                lift_curve[int(prediction.prediction_probas.columns[0])] = {
-                    "lift": [i[8] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "top": [i[0] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "count": [i[1] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "prob": [i[2] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "tp": [i[3] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "fp": [i[4] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "precision": [i[5] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "recall": [i[6] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "f1_score": [i[7] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "max_lift": [i[9] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "relative_lift": [i[10] for i in lift_table[prediction.prediction_probas.columns[0]]],
-                    "percent": lift_table[prediction.prediction_probas.columns[0]][0][11],
-                }
+                # lift_curve[int(prediction.prediction_probas.columns[0])] = LiftCurveData(
+                lift_curve[label] = LiftCurveData(
+                    lift=[i[8] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    top=[i[0] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    count=[i[1] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    prob=[i[2] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    tp=[i[3] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    fp=[i[4] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    precision=[i[5] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    recall=[i[6] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    f1_score=[i[7] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    max_lift=[i[9] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    relative_lift=[i[10] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    percent=[i[11] for i in lift_table[prediction.prediction_probas.columns[0]]],
+                    # percent = lift_table[prediction.prediction_probas.columns[0]][0][11],
+                )
         return lift_curve
 
 
 @default_renderer(wrap_type=ClassificationLiftCurve)
 class ClassificationLiftCurveRenderer(MetricRenderer):
     def render_html(self, obj: ClassificationLiftCurve) -> List[BaseWidgetInfo]:
-        current_lift_curve = obj.get_result().current_lift_curve
-        reference_lift_curve = obj.get_result().reference_lift_curve
+        current_lift_curve: Optional[LiftCurve] = obj.get_result().current_lift_curve
+        reference_lift_curve: Optional[LiftCurve] = obj.get_result().reference_lift_curve
         if current_lift_curve is None:
             return []
 

diff --git a/src/evidently/metrics/classification_performance/lift_table_metric.py b/src/evidently/metrics/classification_performance/lift_table_metric.py
@@ -1,15 +1,23 @@
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Type
+from typing import Union
 
 import pandas as pd
 
+from evidently._pydantic_compat import BaseModel
 from evidently.base_metric import InputData
+from evidently.base_metric import Metric
 from evidently.base_metric import MetricResult
-from evidently.calculations.classification_performance import PredictionData
 from evidently.calculations.classification_performance import calculate_lift_table
 from evidently.calculations.classification_performance import get_prediction_data
-from evidently.metrics.base_metric import Metric
+from evidently.metric_results import Label
+from evidently.metric_results import PredictionData
 from evidently.model.widget import BaseWidgetInfo
+from evidently.options.base import AnyOptions
 from evidently.renderers.base_renderer import MetricRenderer
 from evidently.renderers.base_renderer import default_renderer
 from evidently.renderers.html_widgets import TabData
@@ -18,10 +26,26 @@
 from evidently.renderers.html_widgets import widget_tabs
 from evidently.utils.data_operations import process_columns
 
+if TYPE_CHECKING:
+    from evidently._pydantic_compat import Model
+
+
+class LabelModel(BaseModel):
+    __root__: Union[int, str]
+
+    def validate(cls: Type["Model"], value: Any):  # type: ignore[override, misc]
+        try:
+            return int(value)
+        except TypeError:
+            return value
+
+
+LiftTable = Dict[Union[LabelModel, Label], List[List[Union[float, int]]]]
+
 
 class ClassificationLiftTableResults(MetricResult):
-    current_lift_table: Optional[dict] = None
-    reference_lift_table: Optional[dict] = None
+    current_lift_table: Optional[LiftTable] = None
+    reference_lift_table: Optional[LiftTable] = None
     top: Optional[int] = 10
 
 
@@ -38,8 +62,9 @@ class ClassificationLiftTable(Metric[ClassificationLiftTableResults]):
 
     top: int
 
-    def __init__(self, top: int = 10) -> None:
+    def __init__(self, top: int = 10, options: AnyOptions = None) -> None:
         self.top = top
+        super().__init__(options=options)
 
     def calculate(self, data: InputData) -> ClassificationLiftTableResults:
         dataset_columns = process_columns(data.current_data, data.column_mapping)
@@ -136,7 +161,7 @@ def render_html(self, obj: ClassificationLiftTable) -> List[BaseWidgetInfo]:
                         title="",
                         size=size,
                     )
-                    tab_data.append(TabData(label, table))
+                    tab_data.append(TabData(str(label), table))
                 result.append(widget_tabs(title="Current: Lift Table", tabs=tab_data))
         if reference_lift_table is not None:
             if len(reference_lift_table.keys()) == 1:
@@ -157,6 +182,6 @@ def render_html(self, obj: ClassificationLiftTable) -> List[BaseWidgetInfo]:
                         title="",
                         size=size,
                     )
-                    tab_data.append(TabData(label, table))
+                    tab_data.append(TabData(str(label), table))
                 result.append(widget_tabs(title="Reference: Lift Table", tabs=tab_data))
         return result
diff --git a/src/evidently/renderers/html_widgets.py b/src/evidently/renderers/html_widgets.py
@@ -15,6 +15,7 @@
 
 from evidently.metric_results import Distribution
 from evidently.metric_results import HistogramData
+from evidently.metric_results import LiftCurve
 from evidently.metric_results import PRCurve
 from evidently.metric_results import ROCCurve
 from evidently.model.widget import BaseWidgetInfo
@@ -729,8 +730,8 @@ def get_pr_rec_plot_data(
 
 
 def get_lift_plot_data(
-    current_lift_curve: dict,
-    reference_lift_curve: Optional[dict],
+    current_lift_curve: LiftCurve,
+    reference_lift_curve: Optional[PRCurve],
     color_options: ColorOptions,
 ) -> List[Tuple[str, BaseWidgetInfo]]:
     """
@@ -759,14 +760,13 @@ def get_lift_plot_data(
     for label in current_lift_curve.keys():
         fig = make_subplots(rows=1, cols=cols, subplot_titles=subplot_titles, shared_yaxes=True)
         trace = go.Scatter(
-            x=current_lift_curve[label]["top"],
-            y=current_lift_curve[label]["lift"],
+            x=current_lift_curve[label].top,
+            y=current_lift_curve[label].lift,
             mode="lines+markers",
             name="Lift",
             hoverinfo="text",
             text=[
-                f"top: {str(int(current_lift_curve[label]['top'][i]))}, "
-                f"lift={str(current_lift_curve[label]['lift'][i])}"
+                f"top: {str(int(current_lift_curve[label].top[i]))}, " f"lift={str(current_lift_curve[label].lift[i])}"
                 for i in range(100)
             ],
             legendgroup="Lift",
@@ -779,14 +779,14 @@ def get_lift_plot_data(
         fig.update_xaxes(title_text="Top", row=1, col=1)
         if reference_lift_curve is not None:
             trace = go.Scatter(
-                x=reference_lift_curve[label]["top"],
-                y=reference_lift_curve[label]["lift"],
+                x=reference_lift_curve[label].top,
+                y=reference_lift_curve[label].lift,
                 mode="lines+markers",
                 name="Lift",
                 hoverinfo="text",
                 text=[
-                    f"top: {str(int(reference_lift_curve[label]['top'][i]))}, "
-                    f"lift={str(reference_lift_curve[label]['lift'][i])}"
+                    f"top: {str(int(reference_lift_curve[label].top[i]))}, "
+                    f"lift={str(reference_lift_curve[label].lift[i])}"
                     for i in range(100)
                 ],
                 legendgroup="Lift",