Skip to content

FIX: Add microseconds to the webpoda queries #292

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
greglucas merged 2 commits into from
Jan 20, 2026
Merged

FIX: Add microseconds to the webpoda queries #292

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a9612c
FIX: Add microseconds to the webpoda queries
greglucas Jan 20, 2026
9c01949
MNT: Change to an exclusive right-edge for querying webpoda
greglucas Jan 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 10 additions & 11 deletions imap_data_access/webpoda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,8 @@ def get_packet_times_ert(
# are handled correctly, so pass them as params
params = (
# Query the ERT field between start and end date
f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"&ert<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
f"&ert<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
# only get the time (packet time)
# Represent all times in yyyy-MM-dd'T'HH:mm:ss format
"&project(time)&formatTime(\"yyyy-MM-dd'T'HH:mm:ss\")"
Expand Down Expand Up @@ -245,8 +245,8 @@ def get_packet_binary_data_sctime(
query_range = f"{WEBPODA_APID_URL}/{SYSTEM_ID}/apid_{apid}.bin"
params = (
# Query the SCT field between start and end date
f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"&time<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
f"&time<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
# only the raw packet data
"&project(packet)"
)
Expand Down Expand Up @@ -303,7 +303,7 @@ def download_daily_data(
logger.info(f"Unique spacecraft dates with packets: {unique_dates}")

# Iterate over the packet dates to make a query for each individual spacecraft day
# packet_date 00:00:00 -> packet_date 23:59:59
# packet_date 00:00:00 -> packet_date+1 00:00:00
for date in unique_dates:
path = _get_latest_version_file_path(
instrument=instrument,
Expand All @@ -314,7 +314,7 @@ def download_daily_data(
continue

daily_start_time = datetime.datetime.combine(date, datetime.time.min)
daily_end_time = datetime.datetime.combine(date, datetime.time.max)
daily_end_time = daily_start_time + datetime.timedelta(days=1)

# Some instruments request a buffer of packets on either side of the midnight
# boundary to ensure their packet groupings work together.
Expand Down Expand Up @@ -439,13 +439,12 @@ def download_repointing_data(
f"{packet_times[-1]}, skipping"
)
continue
# NOTE: All queries are <= / >= following this, so we need to make sure we
# are not double grabbing packets into the pointings.
# The times included are [repointing_start, repointing_end), exclusive
# on the right edge
# NOTE: We need to make sure we are not double grabbing packets into the
# pointings. The times included are [repointing_start, repointing_end),
# exclusive on the right edge
pointing_end = datetime.datetime.strptime(
repointings[i + 1]["repoint_end_utc"], "%Y-%m-%d %H:%M:%S.%f"
) - datetime.timedelta(seconds=1)
)
if pointing_end < packet_times[0]:
# This pointing is before the first packet time, so skip it
logger.debug(
Expand Down
14 changes: 7 additions & 7 deletions tests/test_webpoda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ def test_get_packet_times_ert(mock_send_request, mock_request):
f"https://lasp.colorado.edu/ops/imap/poda/dap2/apids/SID1/apid_{apid}.txt",
headers={"Authorization": "Basic test_token"},
params=(
f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"&ert<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"ert>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
f"&ert<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
"&project(time)&formatTime(\"yyyy-MM-dd'T'HH:mm:ss\")"
),
)
Expand All @@ -60,7 +60,7 @@ def test_get_packet_binary_data_sctime(mock_send_request, mock_request):
mock_send_request.return_value = mock_response

start_time = datetime.datetime(2024, 12, 1, 0, 0, 0)
end_time = datetime.datetime(2024, 12, 1, 23, 59, 59)
end_time = datetime.datetime(2024, 12, 1, 23, 59, 59, 999999)
apid = 1136

result = get_packet_binary_data_sctime(apid, start_time, end_time)
Expand All @@ -71,8 +71,8 @@ def test_get_packet_binary_data_sctime(mock_send_request, mock_request):
f"https://lasp.colorado.edu/ops/imap/poda/dap2/apids/SID1/apid_{apid}.bin",
headers={"Authorization": "Basic test_token"},
params=(
f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"&time<={end_time.strftime('%Y-%m-%dT%H:%M:%S')}"
f"time>={start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
f"&time<{end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
"&project(packet)"
),
)
Expand Down Expand Up @@ -113,8 +113,8 @@ def test_download_daily_data(
assert call == (
1184,
start_time - datetime.timedelta(minutes=1),
datetime.datetime.combine(start_time, datetime.time.max)
+ datetime.timedelta(minutes=1),
# end time + 1 day, then buffer of 1 minute
start_time + datetime.timedelta(days=1) + datetime.timedelta(minutes=1),
)

# We expect two daily files to be created because we have packets
Expand Down