ManifoldRG · helenlu66 · Nov 11, 2025 · Nov 15, 2025 · Nov 15, 2025 · Nov 15, 2025
diff --git a/src/v1/results/amae_comparison.pdf b/src/v1/results/amae_comparison.pdf
diff --git a/src/v1/results/confusion_matrix_gpt-5_piqa.pdf b/src/v1/results/confusion_matrix_gpt-5_piqa.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_AerialMaritimeDrone.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_AerialMaritimeDrone.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_Aquarium.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_Aquarium.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_BCCD.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_BCCD.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_ChessPieces.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_ChessPieces.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_DroneControl.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_DroneControl.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_EgoHands.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_EgoHands.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_HardHatWorkers.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_HardHatWorkers.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_MaskWearing.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_MaskWearing.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_PKLot.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_PKLot.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_ShellfishOpenImages.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_ShellfishOpenImages.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_ThermalCheetah.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_ThermalCheetah.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_UnoCards.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_UnoCards.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_VehiclesOpenImages.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_VehiclesOpenImages.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_brackishUnderwater.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_brackishUnderwater.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_dice.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_dice.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_selfdrivingCar.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_selfdrivingCar.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_thermalDogsAndPeople.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_thermalDogsAndPeople.pdf
diff --git a/src/v1/results/confusion_matrix_gpt5_odinw_websiteScreenshots.pdf b/src/v1/results/confusion_matrix_gpt5_odinw_websiteScreenshots.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_AerialMaritimeDrone.pdf b/src/v1/results/confusion_matrix_magma_odinw_AerialMaritimeDrone.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_Aquarium.pdf b/src/v1/results/confusion_matrix_magma_odinw_Aquarium.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_BCCD.pdf b/src/v1/results/confusion_matrix_magma_odinw_BCCD.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_ChessPieces.pdf b/src/v1/results/confusion_matrix_magma_odinw_ChessPieces.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_DroneControl.pdf b/src/v1/results/confusion_matrix_magma_odinw_DroneControl.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_EgoHands.pdf b/src/v1/results/confusion_matrix_magma_odinw_EgoHands.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_HardHatWorkers.pdf b/src/v1/results/confusion_matrix_magma_odinw_HardHatWorkers.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_MaskWearing.pdf b/src/v1/results/confusion_matrix_magma_odinw_MaskWearing.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_PKLot.pdf b/src/v1/results/confusion_matrix_magma_odinw_PKLot.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_ShellfishOpenImages.pdf b/src/v1/results/confusion_matrix_magma_odinw_ShellfishOpenImages.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_ThermalCheetah.pdf b/src/v1/results/confusion_matrix_magma_odinw_ThermalCheetah.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_UnoCards.pdf b/src/v1/results/confusion_matrix_magma_odinw_UnoCards.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_VehiclesOpenImages.pdf b/src/v1/results/confusion_matrix_magma_odinw_VehiclesOpenImages.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_brackishUnderwater.pdf b/src/v1/results/confusion_matrix_magma_odinw_brackishUnderwater.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_dice.pdf b/src/v1/results/confusion_matrix_magma_odinw_dice.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_selfdrivingCar.pdf b/src/v1/results/confusion_matrix_magma_odinw_selfdrivingCar.pdf
diff --git a/src/v1/results/confusion_matrix_magma_odinw_thermalDogsAndPeople.pdf b/src/v1/results/confusion_matrix_magma_odinw_thermalDogsAndPeople.pdf
diff --git a/src/v1/results/confusion_matrix_magma_piqa.pdf b/src/v1/results/confusion_matrix_magma_piqa.pdf
diff --git a/src/v1/results/data_analysis.ipynb b/src/v1/results/data_analysis.ipynb
diff --git a/src/v1/results/data_analysis.py b/src/v1/results/data_analysis.py
@@ -104,8 +104,12 @@ def barplot(dataframe, title, ylabel, xlabel, save_path, y='Exact Match Rate', y
 }
 
 #%% load Pi-0 results
-with open('./pi0/pi0_base_openx_results_final.json') as f:
-    pi0_base_openx = json.load(f)
+try:
+    with open('./pi0/pi0_base_openx_results_final.json') as f:
+        pi0_base_openx = json.load(f)
+except FileNotFoundError:
+    with open('./pi0/pi0_base_openx_results.json') as f:
+        pi0_base_openx = json.load(f)
 with open('./pi0/pi0_base_overcooked_results.json') as f:
     pi0_base_overcooked = json.load(f)
 with open('./pi0/pi0_hf_bfcl_inference_results.json') as f:
@@ -185,7 +189,7 @@ def barplot(dataframe, title, ylabel, xlabel, save_path, y='Exact Match Rate', y
 
 pi0_bfcl_emr = extract_key_from_json(pi0_hf_bfcl_inference, 'exact_match_accuracy')
 magma_bfcl_emr = extract_key_from_json(magma_bfcl, 'exact_match_accuracy')
-gpt5_bfcl_emr = [0.285]  # referenced from literature
+gpt5_bfcl_emr = None
 log_count('Pi-0 BFCL exact match', pi0_bfcl_emr)
 log_count('Magma BFCL exact match', magma_bfcl_emr)
 

diff --git a/src/v1/results/emr_comparison.pdf b/src/v1/results/emr_comparison.pdf
diff --git a/src/v1/results/f1_comparison.pdf b/src/v1/results/f1_comparison.pdf
diff --git a/src/v1/results/macro_f1_comparison.pdf b/src/v1/results/macro_f1_comparison.pdf
diff --git a/src/v1/results/macro_precision_comparison.pdf b/src/v1/results/macro_precision_comparison.pdf
diff --git a/src/v1/results/macro_recall_comparison.pdf b/src/v1/results/macro_recall_comparison.pdf
diff --git a/src/v1/results/openx_namse_comparison.pdf b/src/v1/results/openx_namse_comparison.pdf
diff --git a/src/v1/results/precision_comparison.pdf b/src/v1/results/precision_comparison.pdf
diff --git a/src/v1/results/recall_comparison.pdf b/src/v1/results/recall_comparison.pdf
diff --git a/src/v1/results/similarity_score_comparison.pdf b/src/v1/results/similarity_score_comparison.pdf