fix handling values out of bounds in pandas

apps/api/src/python/query/bigquery.ts

@@ -84,12 +84,12 @@ def _briefer_make_bq_query():
         for col, bq_type in columns_by_type.items():
             if col in df.columns:
                 if bq_type in ['DATE', 'DATETIME']:
-                    df[col] = pd.to_datetime(df[col], errors='coerce')
+                    df[col] = pd.to_datetime(df[col], errors='coerce', unit='ms')
                 elif bq_type == 'TIMESTAMP':
                     # Convert to timezone-aware datetime, preserving the original time zone
-                    df[col] = pd.to_datetime(df[col], errors='coerce', utc=True).dt.tz_convert(None)
+                    df[col] = pd.to_datetime(df[col], errors='coerce', utc=True, unit='ms').dt.tz_convert(None)
                 elif bq_type == 'TIME':
-                    df[col] = pd.to_datetime(df[col], errors='coerce').dt.time
+                    df[col] = pd.to_datetime(df[col], errors='coerce', unit='ms').dt.time
                 elif bq_type == 'NUMERIC':
                     df[col] = pd.to_numeric(df[col], errors='coerce')
 
@@ -158,7 +158,14 @@ def _briefer_make_bq_query():
             return df
 
         bq_storage_client = bigquery_storage.BigQueryReadClient(credentials=credentials)
-        df_iter = query_result.to_dataframe_iterable(bqstorage_client=bq_storage_client)
+
+        # handle timestamp precision to avoid OutOfBoundsDatetime
+        timestamp_columns = [field.name for field in schema if field.field_type == 'TIMESTAMP' or field.field_type == 'DATETIME']
+        print(json.dumps({"type": "log", "message": f"Timestamp columns: {timestamp_columns}"}))
+        dtype_dict = {col: 'datetime64[ms]' for col in timestamp_columns}
+        print(json.dumps({"type": "log", "message": f"Timestamp columns: {dtype_dict}"}))
+        df_iter = query_result.to_dataframe_iterable(bqstorage_client=bq_storage_client, dtypes=dtype_dict)
+
         df = pd.DataFrame()
 
         initial_rows = []