Check filtering step

astrolabsoftware · Jun 11, 2024 · c10a8ef · c10a8ef
1 parent b83e397
commit c10a8ef
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/bin/raw2science.py b/bin/raw2science.py
@@ -82,6 +82,21 @@ def main():
             latestfirst=False
         )
 
+    # DEBUG STATEMENTS
+    df_static = spark.read.format('parquet').load(
+        rawdatapath + "/year={}/month={}/day={}".format(
+            args.night[0:4],
+            args.night[4:6],
+            args.night[6:8]
+        )
+    )
+
+    print("COUNT BEFORE: {}".format(df_static.count()))
+    df_filtered = df_static\
+        .filter(df_static['candidate.nbad'] == 0)\
+        .filter(df_static['candidate.rb'] >= 0.55)
+    print("COUNT AFTER: {}".format(df_filtered.count()))
+
     # Apply science modules
     if 'candidate' in df.columns:
         # Apply quality cuts