Metrics redesign (#1389)

evidentlyai · Jan 10, 2025 · beec1e5 · beec1e5
1 parent 2b7a2d1
commit beec1e5
Show file tree

Hide file tree

Showing 97 changed files with 21,608 additions and 325 deletions.
diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml
@@ -162,7 +162,7 @@ jobs:
 
       - name: Wait UI to be ready to test
         working-directory: ui/service
-        run: pnpm wait-on tcp:127.0.0.1:8000 -t 4m
+        run: pnpm wait-on tcp:127.0.0.1:8000 -t 6m
 
       - name: Run Service Playwright tests
         working-directory: ui/service

diff --git a/examples/cookbook/descriptors.ipynb b/examples/cookbook/descriptors.ipynb
diff --git a/examples/future_dashboads.ipynb b/examples/future_dashboads.ipynb
@@ -0,0 +1,137 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "initial_id",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-01-09T23:58:58.734191Z",
+     "start_time": "2025-01-09T23:58:58.566108Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from random import randint\n",
+    "\n",
+    "from evidently.ui.dashboards.reports import DistributionPanel\n",
+    "from evidently.future.datasets import BinaryClassification\n",
+    "from evidently.future.metrics import PrecisionByLabel\n",
+    "from evidently.future.metrics.column_statistics import CategoryCount\n",
+    "from evidently.ui.dashboards import CounterAgg\n",
+    "from evidently.ui.dashboards import PanelValue\n",
+    "from evidently.ui.dashboards import ReportFilter\n",
+    "import datetime\n",
+    "import pandas as pd\n",
+    "\n",
+    "from evidently.future.metrics import MeanValue\n",
+    "from evidently.future.metrics import UniqueValueCount\n",
+    "from evidently.future.datasets import Dataset\n",
+    "from evidently.future.datasets import DataDefinition\n",
+    "from evidently.future.datasets import ColumnInfo\n",
+    "from evidently.future.datasets import ColumnType\n",
+    "\n",
+    "from evidently.future.backport import snapshot_v2_to_v1\n",
+    "from evidently.future.backport import SingleValueDashboardPanel\n",
+    "from evidently.future.report import Report\n",
+    "from evidently.future.tests import lte\n",
+    "from evidently.ui.dashboards import DashboardPanelCounter\n",
+    "from evidently.ui.workspace import Workspace\n",
+    "\n",
+    "def create_snapshot(i):\n",
+    "    df = pd.DataFrame({\n",
+    "        \"col\": list(randint(0, 5) for _ in range(i + 5)),\n",
+    "        \"target\": list(x % 2 for x in range(i + 5)),\n",
+    "        \"prediction\": list(x / 10 for x in range(i + 5)),\n",
+    "    })\n",
+    "    dataset = Dataset.from_pandas(\n",
+    "        df,\n",
+    "        data_definition=DataDefinition(\n",
+    "            numerical_columns=[\"col\"],\n",
+    "            classification=[BinaryClassification()]\n",
+    "        ),\n",
+    "    )\n",
+    "    report = Report([MeanValue(column=\"col\", tests=[lte(4)]), UniqueValueCount(column=\"col\")])\n",
+    "    snapshot_v2 = report.run(dataset, None)\n",
+    "\n",
+    "    snapshot_v1 = snapshot_v2_to_v1(snapshot_v2)\n",
+    "    import uuid6\n",
+    "\n",
+    "    snapshot_v1.id = uuid6.UUID(int=i, version=7)\n",
+    "    snapshot_v1.timestamp = datetime.datetime.now() - datetime.timedelta(days=1)\n",
+    "    return snapshot_v1\n",
+    "\n",
+    "ws = Workspace.create(\"./workspace\")\n",
+    "from evidently.ui.type_aliases import ZERO_UUID\n",
+    "\n",
+    "project = ws.get_project(ZERO_UUID)\n",
+    "if project is None:\n",
+    "    from evidently.ui.base import Project\n",
+    "\n",
+    "    project = ws.add_project(Project(id=ZERO_UUID, name=\"test proj\"))\n",
+    "\n",
+    "    project.dashboard.add_panel(SingleValueDashboardPanel(metric_id=MeanValue(column=\"col\", tests=[lte(4)]).metric_id))\n",
+    "    project.dashboard.add_panel(DashboardPanelCounter(\n",
+    "        title=\"Last mean value\",\n",
+    "        agg=CounterAgg.LAST,\n",
+    "        value=PanelValue(field_path=\"value\", metric_args={\"metric.metric_id\": \"2e5caa9690281e02cf243c736d687782\"}),\n",
+    "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
+    "    ))\n",
+    "    project.dashboard.add_panel(DistributionPanel(\n",
+    "        title=\"Distr\",\n",
+    "        value=PanelValue(field_path=\"values\", metric_args={\"metric.type\": \"evidently:metric_v2:UniqueValueCount\"}),\n",
+    "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
+    "    ))\n",
+    "    # project.dashboard.add_panel(SingleValueDashboardPanel(metric_id=\"mean:col\"))\n",
+    "    project.save()\n",
+    "\n",
+    "for i in range(10):\n",
+    "    project.add_snapshot(create_snapshot(i))    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "35a20fd14d577dc8",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-01-09T23:56:41.749035100Z",
+     "start_time": "2025-01-09T23:53:00.368240Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# use `evidently ui` to run UI service to see project dashboard."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "babdaf65-d7a2-454d-9429-e270dd7fffc7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}