Tweaks from David and Scott

Author: epugh

jupyterlite/files/examples/Jaccard and RBO Comparison.ipynb

@@ -23,12 +23,12 @@
   "cells": [
     {
       "cell_type": "markdown",
-      "source": "# Jaccard and RBO Comparison \nTo understand the magnatude of changes to your query result sets, you can compare multiple snapshots together, either from the same case or different cases.   \n\nThis notebook provides both Jaccard and Rank Biased Overlap (RBO) metrics.\n\nPlease copy this example and customize it for your own purposes!",
+      "source": "# Jaccard and RBO Comparison \nTo understand the magnatude of changes to your query result sets, you can compare multiple snapshots to each other.\n\nThis notebook provides both Jaccard and Rank Biased Overlap (RBO) metrics.\n\nPlease copy this example and customize it for your own purposes!",
       "metadata": {}
     },
     {
       "cell_type": "code",
-      "source": "from js import fetch\nfrom typing import List, Optional, Union\n\nimport json\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport piplite\nawait piplite.install('seaborn')\nawait piplite.install('rbo')\n\nimport rbo\nimport seaborn as sns\n\nimport os\n\nos.environ[\"TQDM_DISABLE\"] = \"1\"",
+      "source": "from js import fetch\nfrom typing import List, Optional, Union\n\nimport json\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport piplite\nawait piplite.install('seaborn')\nawait piplite.install('rbo')\n\nimport rbo\nimport seaborn as sns\n\nimport os",
       "metadata": {
         "trusted": true
       },
@@ -72,16 +72,30 @@
         }
       ]
     },
+    {
+      "cell_type": "code",
+      "source": "os.environ[\"TQDM_DISABLE\"] = \"1\"",
+      "metadata": {
+        "trusted": true
+      },
+      "execution_count": 5,
+      "outputs": []
+    },
     {
       "cell_type": "code",
       "source": "def jaccard(l1, l2, max_n):\n    if len(l1) == 0 and len(l2) == 0:\n        return 1\n    max_len = min(len(l1), len(l2), max_n)\n    set1 = set(l1[:max_len])\n    set2 = set(l2[:max_len])\n    intersection = len(set1.intersection(set2))\n    union = len(set1) + len(set2) - intersection\n    return float(intersection) / union\n\nasync def load_snapshots(case_id1, snapshot_id1, case_id2, snapshot_id2):\n    df_a = await load_snapshot(case_id1, snapshot_id1)\n    df_b = await load_snapshot(case_id2, snapshot_id2)\n    return df_a.merge(df_b, on='query')\n\nasync def compare(case_id1, snapshot_id1, case_id2, snapshot_id2):\n    df = await load_snapshots(case_id1, snapshot_id1, case_id2, snapshot_id2)\n    \n    df['jaccard'] = df.apply(lambda row: jaccard(row['docs_x'], row['docs_y'], 10), axis=1)\n    df['rbo'] = df.apply(lambda row: rbo.RankingSimilarity(row['docs_x'], row['docs_y']).rbo(), axis=1)\n    df['score_delta'] = df['score_y'] - df['score_x']\n    df.name = f\"Case {case_id1} snapshot {snapshot_id1} vs. case {case_id1} snapshot {snapshot_id2}\"\n    return df\n\n\n\nawait compare(case_id1=6789, snapshot_id1=2471, case_id2=6789, snapshot_id2=2472)",
       "metadata": {
         "trusted": true
       },
-      "execution_count": 7,
+      "execution_count": 6,
       "outputs": [
         {
-          "execution_count": 7,
+          "name": "stderr",
+          "text": "/lib/python3.11/site-packages/rbo/rbo.py:129: TqdmMonitorWarning: tqdm:disabling monitor support (monitor_interval = 0) due to:\ncan't start new thread\n  for d in tqdm(range(1, k), disable=~self.verbose):\n",
+          "output_type": "stream"
+        },
+        {
+          "execution_count": 6,
           "output_type": "execute_result",
           "data": {
             "text/plain": "                    num_results_x  score_x  \\\nquery                                        \nprojector screen                1      1.0   \nnotebook                        1      1.0   \niphone 8                        1      1.0   \nprinter                         1      1.0   \ncomputer                        1      1.0   \n...                           ...      ...   \nwindows 10                      1      1.0   \nmicrowave                       1      1.0   \nbluetooth speakers              1      1.0   \ncoffee                          1      1.0   \nvans                            1      1.0   \n\n                                                               docs_x  \\\nquery                                                                   \nprojector screen    [1069226, 47471, 490523, 1229109, 1229118, 325...   \nnotebook            [3851056, 3959000, 1550833, 1684763, 1675257, ...   \niphone 8            [2048598, 1648546, 79524888, 1857711, 3613408,...   \nprinter             [3849563, 2225354, 1569761, 798960, 377837, 13...   \ncomputer            [560468, 532095, 560475, 523407, 693956, 56047...   \n...                                                               ...   \nwindows 10          [4481689, 3902727, 1560529, 1797902, 3155116, ...   \nmicrowave                       [79513345, 4020048, 1768856, 2936032]   \nbluetooth speakers  [1993197, 3537784, 279672, 2663204, 558184, 33...   \ncoffee              [1996660, 2102472, 79583150, 1357989, 656359, ...   \nvans                [78503576, 79118095, 77388459, 78322005, 79013...   \n\n                    num_results_y  score_y  \\\nquery                                        \nprojector screen                1      1.0   \nnotebook                        1      1.0   \niphone 8                        1      1.0   \nprinter                         1      1.0   \ncomputer                        1      1.0   \n...                           ...      ...   \nwindows 10                      1      1.0   \nmicrowave                       1      1.0   \nbluetooth speakers              1      1.0   \ncoffee                          1      1.0   \nvans                            1      1.0   \n\n                                                               docs_y  \\\nquery                                                                   \nprojector screen    [1069226, 47471, 490523, 1229109, 1229118, 325...   \nnotebook            [3851056, 3959000, 1550833, 1684763, 1675257, ...   \niphone 8            [2048598, 1648546, 79524888, 1857711, 3613408,...   \nprinter             [3849563, 2225354, 1569761, 798960, 377837, 13...   \ncomputer            [560468, 532095, 560475, 523407, 693956, 56047...   \n...                                                               ...   \nwindows 10          [4481689, 3902727, 1560529, 1797902, 3155116, ...   \nmicrowave                       [79513345, 4020048, 1768856, 2936032]   \nbluetooth speakers  [1993197, 3537784, 279672, 2663204, 558184, 33...   \ncoffee              [1996660, 2102472, 79583150, 1357989, 656359, ...   \nvans                [78503576, 79118095, 77388459, 78322005, 79013...   \n\n                    jaccard  rbo  score_delta  \nquery                                          \nprojector screen        1.0  1.0          0.0  \nnotebook                1.0  1.0          0.0  \niphone 8                1.0  1.0          0.0  \nprinter                 1.0  1.0          0.0  \ncomputer                1.0  1.0          0.0  \n...                     ...  ...          ...  \nwindows 10              1.0  1.0          0.0  \nmicrowave               1.0  1.0          0.0  \nbluetooth speakers      1.0  1.0          0.0  \ncoffee                  1.0  1.0          0.0  \nvans                    1.0  1.0          0.0  \n\n[135 rows x 9 columns]",
@@ -93,11 +107,45 @@
     },
     {
       "cell_type": "code",
-      "source": "import matplotlib\nmatplotlib.rc_file_defaults()\n\ndef plot_compare(df):\n    figure, axes = plt.subplots(1, 3, figsize=(10, 4))\n    figure.suptitle(df.name)\n\n    sns.barplot(ax=axes[0], x=df['score_delta'], y=df.index, width=0.3, color='darkgrey')\n    axes[0].set(xlim=(-1, 1))\n    axes[0].set_xlabel('Change in Score')\n    axes[0].set_ylabel('')\n    axes[0].set_facecolor((0.90, 0.90, 0.90))\n    axes[0].grid(True)\n    axes[0].spines['top'].set_visible(False)\n    axes[0].spines['right'].set_visible(False)\n    axes[0].spines['bottom'].set_visible(False)\n    axes[0].spines['left'].set_visible(False)\n    axes[0].set_axisbelow(True)\n    axes[0].xaxis.grid(color='w', linestyle='solid')\n    axes[0].yaxis.grid(color='w', linestyle='solid')\n    \n    sns.heatmap(df[['jaccard']], ax=axes[1], cmap='crest', annot=True, xticklabels=False, yticklabels=False)\n    axes[1].set_xlabel('Jaccard Similiarity')\n    axes[1].set_ylabel('')\n    \n    sns.heatmap(df[['rbo']], ax=axes[2], cmap='crest', annot=True, xticklabels=False, yticklabels=False)\n    axes[2].set_xlabel('Rank Biased Overlap')\n    axes[2].set_ylabel('')\n    \n    plt.show()\n    \ndf = await compare(case_id1=6789, snapshot_id1=2471, case_id2=6789, snapshot_id2=2473)\nplot_compare(df)",
+      "source": "import matplotlib\nmatplotlib.rc_file_defaults()\n\ndef plot_compare(df):\n    figure, axes = plt.subplots(1, 3, figsize=(10, 4))\n    figure.suptitle(df.name)\n\n    sns.barplot(ax=axes[0], x=df['score_delta'], y=df.index, width=0.3, color='darkgrey')\n    axes[0].set(xlim=(-1, 1))\n    axes[0].set_xlabel('Change in Score')\n    axes[0].set_ylabel('')\n    axes[0].set_facecolor((0.90, 0.90, 0.90))\n    axes[0].grid(True)\n    axes[0].spines['top'].set_visible(False)\n    axes[0].spines['right'].set_visible(False)\n    axes[0].spines['bottom'].set_visible(False)\n    axes[0].spines['left'].set_visible(False)\n    axes[0].set_axisbelow(True)\n    axes[0].xaxis.grid(color='w', linestyle='solid')\n    axes[0].yaxis.grid(color='w', linestyle='solid')\n    \n    sns.heatmap(df[['jaccard']], ax=axes[1], cmap='crest', annot=True, xticklabels=False, yticklabels=False)\n    axes[1].set_xlabel('Jaccard Similiarity')\n    axes[1].set_ylabel('')\n    \n    sns.heatmap(df[['rbo']], ax=axes[2], cmap='crest', annot=True, xticklabels=False, yticklabels=False)\n    axes[2].set_xlabel('Rank Biased Overlap')\n    axes[2].set_ylabel('')\n    \n    plt.show()\n    \ndf = await compare(case_id1=6789, snapshot_id1=2471, case_id2=6789, snapshot_id2=2473)\n",
       "metadata": {
         "trusted": true
       },
-      "execution_count": 6,
+      "execution_count": 7,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": "## Overall Jaccard and RBO Scores",
+      "metadata": {}
+    },
+    {
+      "cell_type": "code",
+      "source": "print(f\"Overall Jaccard Score: {df['jaccard'].mean()}\\nOverall RBO Score: {df['rbo'].mean()}\")",
+      "metadata": {
+        "trusted": true
+      },
+      "execution_count": 8,
+      "outputs": [
+        {
+          "name": "stdout",
+          "text": "Overall Jaccard Score: 1.0\nOverall RBO Score: 1.0\n",
+          "output_type": "stream"
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": "## Query Level Jaccard and RBO Scores",
+      "metadata": {}
+    },
+    {
+      "cell_type": "code",
+      "source": "plot_compare(df)",
+      "metadata": {
+        "trusted": true
+      },
+      "execution_count": 9,
       "outputs": [
         {
           "output_type": "display_data",
@@ -111,8 +159,15 @@
     },
     {
       "cell_type": "markdown",
-      "source": "_This notebook was last updated 16-FEB-2024_",
+      "source": "_This notebook was last updated 19-FEB-2024_",
       "metadata": {}
+    },
+    {
+      "cell_type": "code",
+      "source": "",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": []
     }
   ]
 }