Skip to content

Commit 957366b

Browse files
ffuuugorffuuugor
committed
cleanup
1 parent ffc3c32 commit 957366b

File tree

6 files changed

+0
-92
lines changed

6 files changed

+0
-92
lines changed

‎notebooks/additional_setups.ipynb

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -279,48 +279,37 @@
279279
" \"lt_iqr_desc\"\n",
280280
"]\n",
281281
"\n",
282-
"# Initialize list to store results\n",
283282
"all_results = []\n",
284283
"\n",
285-
"# Loop through architectures and collect metrics\n",
286284
"for arch in [\"rn-20\", \"wrn28-2\", \"wrn40-4\"]:\n",
287285
" exp_id = f\"{arch}_CIFAR10\"\n",
288286
" if exp_id not in final_model_metrics:\n",
289287
" continue\n",
290288
" \n",
291-
" # Get precision/recall metrics for this architecture\n",
292289
" results = make_precision_recall_at_k_df_single_threshold(\n",
293290
" scores_df=final_model_metrics[exp_id],\n",
294291
" ground_truth_df=final_model_metrics[exp_id], \n",
295292
" fpr_threshold=0.001,\n",
296293
" k_frac=0.01\n",
297294
" )\n",
298295
" \n",
299-
" # Add architecture column\n",
300296
" results['architecture'] = arch\n",
301297
" \n",
302-
" # Append to list\n",
303298
" all_results.append(results)\n",
304299
"\n",
305-
"# Create column names for precision and recall\n",
306300
"precision_columns = [\"precision_\" + m for m in metrics]\n",
307301
"recall_columns = [\"recall_\" + m for m in metrics]\n",
308302
"\n",
309-
"# Create dataframe with both precision and recall columns\n",
310303
"df = pd.DataFrame.from_records(all_results)[precision_columns + recall_columns + ['architecture']]\n",
311304
"\n",
312-
"# Pivot table to make architectures as columns\n",
313305
"df_pivot = df.melt(id_vars=['architecture'], var_name='metric', value_name='value')\n",
314306
"\n",
315-
"# Create MultiIndex before pivoting\n",
316307
"df_pivot['type'] = df_pivot['metric'].str.split('_').str[0]\n",
317308
"df_pivot['metric'] = df_pivot['metric'].str.split('_').str[1:].str.join('_')\n",
318309
"df_pivot.set_index(['type', 'metric'], inplace=True)\n",
319310
"\n",
320-
"# Now pivot with the MultiIndex\n",
321311
"results_df = df_pivot.pivot(columns='architecture', values='value')\n",
322312
"\n",
323-
"# Sort index to group precision and recall metrics together\n",
324313
"idx = pd.MultiIndex.from_product([['precision', 'recall'], metrics], names=['type', 'metric'])\n",
325314
"results_df = results_df.reindex(idx)\n",
326315
"\n",
@@ -484,48 +473,37 @@
484473
" \"lt_iqr_desc\"\n",
485474
"]\n",
486475
"\n",
487-
"# Initialize list to store results\n",
488476
"all_results = []\n",
489477
"\n",
490-
"# Loop through datasets and collect metrics\n",
491478
"for dataset in [\"CIFAR10\", \"CIFAR100\", \"CINIC10\"]:\n",
492479
" exp_id = f\"wrn28-2_{dataset}\" # Use WRN-28-2 for all datasets\n",
493480
" if exp_id not in final_model_metrics:\n",
494481
" continue\n",
495482
" \n",
496-
" # Get precision/recall metrics for this dataset\n",
497483
" results = make_precision_recall_at_k_df_single_threshold(\n",
498484
" scores_df=final_model_metrics[exp_id],\n",
499485
" ground_truth_df=final_model_metrics[exp_id], \n",
500486
" fpr_threshold=0.001,\n",
501487
" k_frac=0.01\n",
502488
" )\n",
503489
" \n",
504-
" # Add dataset column\n",
505490
" results['dataset'] = dataset\n",
506491
" \n",
507-
" # Append to list\n",
508492
" all_results.append(results)\n",
509493
"\n",
510-
"# Create column names for precision and recall\n",
511494
"precision_columns = [\"precision_\" + m for m in metrics]\n",
512495
"recall_columns = [\"recall_\" + m for m in metrics]\n",
513496
"\n",
514-
"# Create dataframe with both precision and recall columns\n",
515497
"df = pd.DataFrame.from_records(all_results)[precision_columns + recall_columns + ['dataset']]\n",
516498
"\n",
517-
"# Pivot table to make datasets as columns\n",
518499
"df_pivot = df.melt(id_vars=['dataset'], var_name='metric', value_name='value')\n",
519500
"\n",
520-
"# Create MultiIndex before pivoting\n",
521501
"df_pivot['type'] = df_pivot['metric'].str.split('_').str[0]\n",
522502
"df_pivot['metric'] = df_pivot['metric'].str.split('_').str[1:].str.join('_')\n",
523503
"df_pivot.set_index(['type', 'metric'], inplace=True)\n",
524504
"\n",
525-
"# Now pivot with the MultiIndex\n",
526505
"results_df = df_pivot.pivot(columns='dataset', values='value')\n",
527506
"\n",
528-
"# Sort index to group precision and recall metrics together\n",
529507
"idx = pd.MultiIndex.from_product([['precision', 'recall'], metrics], names=['type', 'metric'])\n",
530508
"results_df = results_df.reindex(idx)\n",
531509
"\n",
@@ -628,18 +606,14 @@
628606
}
629607
],
630608
"source": [
631-
"# Initialize results storage\n",
632609
"results = []\n",
633610
"\n",
634-
"# Loop through all experiments\n",
635611
"for exp_id in final_model_metrics.keys():\n",
636612
" scores = final_model_metrics[exp_id][\"lira_score\"]\n",
637613
" labels = final_model_metrics[exp_id][\"target_trained_on\"]\n",
638614
"\n",
639-
" # Calculate LiRA AUC\n",
640615
" lira_auc = roc_auc_score(labels, scores)\n",
641616
"\n",
642-
" # Calculate TPR at FPR=0.001 \n",
643617
" fpr, tpr, thresholds = roc_curve(labels, scores)\n",
644618
" idx = np.argmin(np.abs(fpr - 0.001))\n",
645619
" tpr_at_fpr = tpr[idx]\n",
@@ -650,7 +624,6 @@
650624
" 'tpr_at_fpr': tpr_at_fpr\n",
651625
" })\n",
652626
"\n",
653-
"# Create dataframe\n",
654627
"df = pd.DataFrame(results).set_index('exp_id')\n",
655628
"print(\"\\nLiRA Metrics across all experiments:\")\n",
656629
"df\n"
@@ -813,48 +786,37 @@
813786
" \"lt_iqr_desc\"\n",
814787
"]\n",
815788
"\n",
816-
"# Initialize list to store results\n",
817789
"all_results = []\n",
818790
"\n",
819-
"# Loop through datasets and collect metrics\n",
820791
"for dataset in [\"CIFAR10\", \"CIFAR100\", \"CINIC10\"]:\n",
821792
" exp_id = f\"wrn28-2_{dataset}\" # Use WRN-28-2 for all datasets\n",
822793
" if exp_id not in final_model_metrics:\n",
823794
" continue\n",
824795
" \n",
825-
" # Get precision/recall metrics for this dataset\n",
826796
" results = make_precision_recall_at_k_df_single_threshold(\n",
827797
" scores_df=final_model_metrics[exp_id],\n",
828798
" ground_truth_df=final_model_metrics[exp_id], \n",
829799
" fpr_threshold=0.001,\n",
830800
" k_frac=0.01\n",
831801
" )\n",
832802
" \n",
833-
" # Add dataset column\n",
834803
" results['dataset'] = dataset\n",
835804
" \n",
836-
" # Append to list\n",
837805
" all_results.append(results)\n",
838806
"\n",
839-
"# Create column names for precision and recall\n",
840807
"precision_columns = [\"precision_\" + m for m in metrics]\n",
841808
"recall_columns = [\"recall_\" + m for m in metrics]\n",
842809
"\n",
843-
"# Create dataframe with both precision and recall columns\n",
844810
"df = pd.DataFrame.from_records(all_results)[precision_columns + recall_columns + ['dataset']]\n",
845811
"\n",
846-
"# Pivot table to make datasets as columns\n",
847812
"df_pivot = df.melt(id_vars=['dataset'], var_name='metric', value_name='value')\n",
848813
"\n",
849-
"# Create MultiIndex before pivoting\n",
850814
"df_pivot['type'] = df_pivot['metric'].str.split('_').str[0]\n",
851815
"df_pivot['metric'] = df_pivot['metric'].str.split('_').str[1:].str.join('_')\n",
852816
"df_pivot.set_index(['type', 'metric'], inplace=True)\n",
853817
"\n",
854-
"# Now pivot with the MultiIndex\n",
855818
"results_df = df_pivot.pivot(columns='dataset', values='value')\n",
856819
"\n",
857-
"# Sort index to group precision and recall metrics together\n",
858820
"idx = pd.MultiIndex.from_product([['precision', 'recall'], metrics], names=['type', 'metric'])\n",
859821
"results_df = results_df.reindex(idx)\n",
860822
"\n",
@@ -1065,7 +1027,6 @@
10651027
"for exp_id in final_model_metrics:\n",
10661028
" df = final_model_metrics[exp_id]\n",
10671029
"\n",
1068-
" # Create empty results dictionary with metrics as rows\n",
10691030
" k_fracs = [0.01, 0.03, 0.05, 0.10, 0.20, 0.50]\n",
10701031
"\n",
10711032
" # Populate results\n",
@@ -1080,10 +1041,8 @@
10801041
" results[exp_id][f'k={k_frac*100}% Precision'] = stats['precision_lt_iqr_desc']\n",
10811042
" results[exp_id][f'k={k_frac*100}% Recall'] = stats['recall_lt_iqr_desc']\n",
10821043
"\n",
1083-
" # Convert to DataFrame with metrics as index\n",
10841044
"results_df = pd.DataFrame.from_dict(results, orient='index')\n",
10851045
"\n",
1086-
"# Sort columns to group by k%\n",
10871046
"column_order = []\n",
10881047
"for k_frac in k_fracs:\n",
10891048
" column_order.extend([f'k={k_frac*100}% Precision', f'k={k_frac*100}% Recall'])\n",
@@ -1246,13 +1205,11 @@
12461205
"source": [
12471206
"from collections import defaultdict\n",
12481207
"\n",
1249-
"# Create empty lists/dicts to store results\n",
12501208
"exp_ids = []\n",
12511209
"total_pos = []\n",
12521210
"pos_at_thresh = []\n",
12531211
"max_recalls = defaultdict(list)\n",
12541212
"\n",
1255-
"# Calculate metrics for each experiment\n",
12561213
"for exp_id in final_model_metrics.keys():\n",
12571214
" results = final_model_metrics[exp_id]\n",
12581215
" \n",
@@ -1263,18 +1220,15 @@
12631220
" total_pos.append(total_positives)\n",
12641221
" pos_at_thresh.append(num_positives_at_threshold)\n",
12651222
" \n",
1266-
" # Calculate max recalls for different k%\n",
12671223
" for k in (1, 3, 5, 10, 20, 50):\n",
12681224
" predicted_samples = k * total_positives // 100\n",
12691225
" max_recall = min(1, predicted_samples / num_positives_at_threshold)\n",
12701226
" max_recalls[k].append(max_recall)\n",
12711227
"\n",
1272-
"# Create and display results DataFrame\n",
12731228
"results_dict = {\n",
12741229
" 'Total Positives': total_pos,\n",
12751230
" 'Positives at Threshold': pos_at_thresh,\n",
12761231
"}\n",
1277-
"# Add max recalls to results dictionary\n",
12781232
"for k in max_recalls:\n",
12791233
" results_dict[f'Max Recall {k}%'] = max_recalls[k]\n",
12801234
"\n",
@@ -1407,7 +1361,6 @@
14071361
" \"spearman_lt_iqr\",\n",
14081362
"]\n",
14091363
"\n",
1410-
"# Create DataFrame with metrics for each experiment\n",
14111364
"metrics_by_exp = {}\n",
14121365
"\n",
14131366
"for exp_id in final_model_metrics:\n",
@@ -1422,7 +1375,6 @@
14221375
" \n",
14231376
" metrics_by_exp[exp_id] = metrics_dict\n",
14241377
"\n",
1425-
"# Create DataFrame with experiments as columns and metrics as rows\n",
14261378
"results_df = pd.DataFrame(metrics_by_exp).loc[metrics]\n",
14271379
"\n",
14281380
"display(results_df)\n",

‎notebooks/dp.ipynb

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -133,15 +133,12 @@
133133
"\n",
134134
" df = all_dfs[(clipnorm, noise)]\n",
135135
"\n",
136-
" # Extract the ground truth (target_trained_on) and scores (lira_score)\n",
137136
" y_true = df[\"target_trained_on\"]\n",
138137
" y_scores = df[\"lira_score\"]\n",
139138
"\n",
140-
" # Compute ROC curve and ROC area\n",
141139
" fpr, tpr, thresholds = roc_curve(y_true, y_scores)\n",
142140
" roc_auc = auc(fpr, tpr)\n",
143141
"\n",
144-
" # Plot ROC curve\n",
145142
" plt.plot(fpr, tpr, lw=2,\n",
146143
" color=colors[clipnorm],\n",
147144
" linestyle=linestyles[noise],\n",
@@ -734,7 +731,6 @@
734731
"source": [
735732
"trace_df = all_dfs[(\"10\", \"0\")]\n",
736733
"\n",
737-
"# Get results for different k values\n",
738734
"k_values = [0.01, 0.03, 0.05]\n",
739735
"results = {}\n",
740736
"for k in k_values:\n",
@@ -746,7 +742,6 @@
746742
" )\n",
747743
" results[k] = result\n",
748744
"\n",
749-
"# Create DataFrame with metrics as rows and k values as columns\n",
750745
"metrics_df = pd.DataFrame.from_dict(results, orient='columns')\n",
751746
"metrics_df.columns = [f'k={k}' for k in k_values]\n",
752747
"\n",
@@ -1326,11 +1321,9 @@
13261321
}
13271322
],
13281323
"source": [
1329-
"# Fix noise=0 and k=0.05\n",
13301324
"noise = \"0\"\n",
13311325
"k = 0.05\n",
13321326
"\n",
1333-
"# Get results for different clipnorm values\n",
13341327
"results = {}\n",
13351328
"for clipnorm in [\"1\", \"3\", \"10\"]:\n",
13361329
" if (clipnorm, noise) in all_dfs:\n",
@@ -1343,7 +1336,6 @@
13431336
" )\n",
13441337
" results[clipnorm] = result\n",
13451338
"\n",
1346-
"# Create DataFrame with metrics as rows and clipnorm values as columns\n",
13471339
"metrics_df = pd.DataFrame.from_dict(results, orient='columns')\n",
13481340
"metrics_df.columns = [f'clipnorm={c}' for c in results.keys()]\n",
13491341
"\n",
@@ -1923,12 +1915,10 @@
19231915
}
19241916
],
19251917
"source": [
1926-
"# Fix noise=0 and k=0.05\n",
19271918
"noise = \"0\"\n",
19281919
"k = 0.05\n",
19291920
"fpr_values = [0.1, 0.01, 0.001]\n",
19301921
"\n",
1931-
"# Get results for different fpr thresholds\n",
19321922
"results = {}\n",
19331923
"for fpr in fpr_values:\n",
19341924
" if (clipnorm, noise) in all_dfs:\n",
@@ -1941,7 +1931,6 @@
19411931
" )\n",
19421932
" results[fpr] = result\n",
19431933
"\n",
1944-
"# Create DataFrame with metrics as rows and fpr values as columns\n",
19451934
"metrics_df = pd.DataFrame.from_dict(results, orient='columns')\n",
19461935
"metrics_df.columns = [f'fpr={fpr}' for fpr in fpr_values]\n",
19471936
"\n",

‎notebooks/main_results.ipynb

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,6 @@
301301
" 'Recall on union': recall\n",
302302
" })\n",
303303
"\n",
304-
"# Display as a table\n",
305304
"results_df = pd.DataFrame(results)\n",
306305
"display(results_df)"
307306
]
@@ -482,11 +481,9 @@
482481
" \"shap_desc\",\n",
483482
" \"lt_iqr_desc\",\n",
484483
"]\n",
485-
"# Create empty results dictionary with metrics as rows\n",
486484
"results = {metric: {} for metric in metrics}\n",
487485
"k_fracs = [0.01, 0.03, 0.05, 0.10, 0.20, 0.50]\n",
488486
"\n",
489-
"# Populate results\n",
490487
"for k_frac in k_fracs:\n",
491488
" stats = make_precision_recall_at_k_df_single_threshold(\n",
492489
" scores_df=df,\n",
@@ -496,14 +493,11 @@
496493
" )\n",
497494
" \n",
498495
" for metric in metrics:\n",
499-
" # Add precision and recall columns for each k%\n",
500496
" results[metric][f'k={k_frac*100}% Precision'] = stats[f'precision_{metric}']\n",
501497
" results[metric][f'k={k_frac*100}% Recall'] = stats[f'recall_{metric}']\n",
502498
"\n",
503-
"# Convert to DataFrame with metrics as index\n",
504499
"results_df = pd.DataFrame.from_dict(results, orient='index')\n",
505500
"\n",
506-
"# Sort columns to group by k%\n",
507501
"column_order = []\n",
508502
"for k_frac in k_fracs:\n",
509503
" column_order.extend([f'k={k_frac*100}% Precision', f'k={k_frac*100}% Recall'])\n",
@@ -1079,7 +1073,6 @@
10791073
")\n",
10801074
"\n",
10811075
"plt.grid(True, linestyle='--', alpha=0.7)\n",
1082-
"# plt.title('Correlation between LT-IQR and LiRA Rankings', pad=20, fontsize=14)\n",
10831076
"plt.xlabel(\"LT-IQR Rank\", fontsize=20, labelpad=10)\n",
10841077
"plt.ylabel(\"LiRA Rank\", fontsize=20, labelpad=10)\n",
10851078
"\n",

0 commit comments

Comments
 (0)