From 863f12a4286cb61869a78fb171eca17b2d965044 Mon Sep 17 00:00:00 2001 From: Vyacheslav Morov Date: Mon, 23 Dec 2024 00:02:52 +0100 Subject: [PATCH] Add metrics ROC AUC, f1, precision, recall. --- examples/list_metrics.ipynb | 752 +++++++++++++++++- .../quality_by_class_metric.py | 4 + src/evidently/v2/metrics/data_quality.py | 86 ++ src/evidently/v2/metrics/f1.py | 27 - 4 files changed, 829 insertions(+), 40 deletions(-) create mode 100644 src/evidently/v2/metrics/data_quality.py delete mode 100644 src/evidently/v2/metrics/f1.py diff --git a/examples/list_metrics.ipynb b/examples/list_metrics.ipynb index 4431b30fa7..275255f4ff 100644 --- a/examples/list_metrics.ipynb +++ b/examples/list_metrics.ipynb @@ -6,8 +6,8 @@ "metadata": { "collapsed": true, "ExecuteTime": { - "end_time": "2024-12-19T12:50:08.634926Z", - "start_time": "2024-12-19T12:50:07.210957Z" + "end_time": "2024-12-22T23:01:16.979366Z", + "start_time": "2024-12-22T23:01:15.079542Z" } }, "source": [ @@ -142,13 +142,13 @@ { "metadata": { "ExecuteTime": { - "end_time": "2024-12-19T12:50:09.968018Z", - "start_time": "2024-12-19T12:50:08.727462Z" + "end_time": "2024-12-22T23:01:18.411333Z", + "start_time": "2024-12-22T23:01:17.052241Z" } }, "cell_type": "code", "source": [ - "from evidently.v2.metrics.f1 import F1Metric\n", + "from evidently.v2.metrics.data_quality import F1Metric\n", "from evidently.v2.report import Context\n", "context = Context()\n", "\n", @@ -161,7 +161,7 @@ { "data": { "text/plain": [ - "" + "" ], "text/html": [ "\n", @@ -226,12 +226,12 @@ " </style>\n", " \n", " <script>\n", - " var metric_b77cbdf69e5544a99392929a63e1841c = {"name": "Report", "widgets": [{"type": "table", "title": "F1 metric", "size": 2, "id": "0193def7-eb13-77e8-b9be-03099f4c876b", "details": "", "alertsPosition": null, "alertStats": null, "params": {"header": ["Label", "Value"], "data": [["0", "0.4"], ["1", "0.5714285714285714"]]}, "insights": [], "alerts": [], "tabs": [], "widgets": [], "pageSize": 5, "source_fingerprint": null}]};\n", - " var additional_graphs_metric_b77cbdf69e5544a99392929a63e1841c = {};\n", + " var metric_49a2fe5d0415438a80502f689f1d99fa = {"name": "Report", "widgets": [{"type": "table", "title": "F1 metric", "size": 2, "id": "0193f09a-8391-7935-872d-338170c27596", "details": "", "alertsPosition": null, "alertStats": null, "params": {"header": ["Label", "Value"], "data": [["0", "0.4"], ["1", "0.5714285714285714"]]}, "insights": [], "alerts": [], "tabs": [], "widgets": [], "pageSize": 5, "source_fingerprint": null}]};\n", + " var additional_graphs_metric_49a2fe5d0415438a80502f689f1d99fa = {};\n", "</script>\n", " </head>\n", " <body>\n", - " <div id="root_metric_b77cbdf69e5544a99392929a63e1841c">\n", + " <div id="root_metric_49a2fe5d0415438a80502f689f1d99fa">\n", " </div>\n", " <script>var global = globalThis</script>\n", " <script>var fZ=Object.defineProperty;var pZ=(e,t,n)=>t in e?fZ(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n;var Cs=(e,t,n)=>(pZ(e,typeof t!="symbol"?t+"":t,n),n);function hZ(e,t){for(var n=0;n<t.length;n++){const r=t[n];if(typeof r!="string"&&!Array.isArray(r)){for(const l in r)if(l!=="default"&&!(l in e)){const i=Object.getOwnPropertyDescriptor(r,l);i&&Object.defineProperty(e,l,i.get?i:{enumerable:!0,get:()=>r[l]})}}}return Object.freeze(Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}))}(function(){const t=document.createElement("link").relList;if(t&&t.supports&&t.supports("modulepreload"))return;for(const l of document.querySelectorAll('link[rel="modulepreload"]'))r(l);new MutationObserver(l=>{for(const i of l)if(i.type==="childList")for(const o of i.addedNodes)o.tagName==="LINK"&&o.rel==="modulepreload"&&r(o)}).observe(document,{childList:!0,subtree:!0});function n(l){const i={};return l.integrity&&(i.integrity=l.integrity),l.referrerPolicy&&(i.referrerPolicy=l.referrerPolicy),l.crossOrigin==="use-credentials"?i.credentials="include":l.crossOrigin==="anonymous"?i.credentials="omit":i.credentials="same-origin",i}function r(l){if(l.ep)return;l.ep=!0;const i=n(l);fetch(l.href,i)}})();var $o=typeof globalThis<"u"?globalThis:typeof window<"u"?window:typeof global<"u"?global:typeof self<"u"?self:{};function dr(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}function er(e){if(e.__esModule)return e;var t=e.default;if(typeof t=="function"){var n=function r(){return this instanceof r?Reflect.construct(t,arguments,this.constructor):t.apply(this,arguments)};n.prototype=t.prototype}else n={};return Object.defineProperty(n,"__esModule",{value:!0}),Object.keys(e).forEach(function(r){var l=Object.getOwnPropertyDescriptor(e,r);Object.defineProperty(n,r,l.get?l:{enumerable:!0,get:function(){return e[r]}})}),n}var z$={exports:{}},_1={},B$={exports:{}},sr={};/**\n", @@ -837,9 +837,9 @@ "</script>\n", " \n", " <script>\n", - " window.drawDashboard(metric_b77cbdf69e5544a99392929a63e1841c,\n", - " new Map(Object.entries(additional_graphs_metric_b77cbdf69e5544a99392929a63e1841c)),\n", - " "root_metric_b77cbdf69e5544a99392929a63e1841c"\n", + " window.drawDashboard(metric_49a2fe5d0415438a80502f689f1d99fa,\n", + " new Map(Object.entries(additional_graphs_metric_49a2fe5d0415438a80502f689f1d99fa)),\n", + " "root_metric_49a2fe5d0415438a80502f689f1d99fa"\n", " );\n", " </script>\n", "\n", @@ -856,13 +856,739 @@ ], "execution_count": 2 }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-22T23:01:18.699070Z", + "start_time": "2024-12-22T23:01:18.604254Z" + } + }, + "cell_type": "code", + "source": [ + "from evidently.v2.report import Report\n", + "\n", + "from evidently.v2.metrics.data_quality import F1Metric\n", + "from evidently.v2.metrics.data_quality import PrecisionMetric\n", + "from evidently.v2.metrics.data_quality import RecallMetric\n", + "from evidently.v2.metrics.data_quality import RocAucMetric\n", + "\n", + "report = Report([\n", + " F1Metric(probas_threshold=0.4),\n", + " PrecisionMetric(probas_threshold=0.4),\n", + " RecallMetric(probas_threshold=0.4),\n", + " RocAucMetric(probas_threshold=0.4),\n", + "])\n", + "\n", + "snapshot = report.run(dataset, None)\n", + "snapshot\n" + ], + "id": "8133418b5d2d9f44", + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + " \n", + " \n", + " \n", + "