Skip to content

Implemented date filtering for research in archive, improved logging and error handling #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
victor5lm merged 4 commits into from
Jun 22, 2026
Merged

Implemented date filtering for research in archive, improved logging and error handling #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fdb28ba
Fixed archive module for propper logging and filtering in research
victor5lm Jun 22, 2026
d812fff
Fixed archive unzip in utils
victor5lm Jun 22, 2026
3800562
Updated CHANGELOG.md
victor5lm Jun 22, 2026
edcb648
black
victor5lm Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

- Improved clean module so that an aux cleaning file is employed for specific services if applicable [#692](https://github.com/BU-ISCIII/buisciii-tools/pull/692).
- Improved HTTP error handling in drylab_api module and fixed fix-permissions so that it does not break given insufficient permissions [#694](https://github.com/BU-ISCIII/buisciii-tools/pull/694).
- Implemented date filtering for research in archive, improved logging and error handling [#700](https://github.com/BU-ISCIII/buisciii-tools/pull/700)

#### Fixes

Expand Down
137 changes: 96 additions & 41 deletions buisciii/archive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import shutil
import sys
from math import pow
from datetime import datetime

import rich
import sysrsync
Expand Down Expand Up @@ -159,13 +160,11 @@ def __init__(
and (service_id is None)
and (services_file is None)
):
if self.ser_type == "services_and_colaborations":
if self.ser_type in ["services_and_colaborations", "research"]:
prompt_response = buisciii.utils.prompt_selection(
"Search services by date, or by service ID?",
["Search by date", "Service ID"],
)
else:
prompt_response = "Service ID"

log.info("Services chosen by: " + prompt_response)
if prompt_response == "Search by date":
Expand Down Expand Up @@ -229,37 +228,84 @@ def __init__(
if (self.date_from is not None) and (self.date_until is not None):
stderr.print("Asking our trusty API about selected services")
log.info("Asking our trusty API about selected services")
try:
for service in rest_api.get_request(
request_info="services",
safe=True,
state="delivered",
date_from=str(self.date_from),
date_until=str(self.date_until),
):
self.services[service["service_request_number"]] = {
key: value for key, value in dictionary_template.items()
}
self.services[service["service_request_number"]][
"found_in_system"
] = True
self.services[service["service_request_number"]][
"delivery_date"
] = service["service_delivered_date"]

log.info(f"Services found in the time interval: {len(self.services)}")
log.info(
"Names of the services found in said interval:"
f"{','.join([service for service in self.services.keys()])}"
)
except TypeError:
stderr.print(
"Could not connect to the API (wrong password?)", style="red"

if self.ser_type == "research":
research_base = os.path.join(
conf.get_configuration("global")["data_path"],
self.ser_type,
)
log.error(
"ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection."
stderr.print(f"Scanning directory: {research_base}")
log.info(f"Scanning research directory: {research_base}")

date_from_ts = datetime.strptime(
str(self.date_from), "%Y-%m-%d"
).timestamp()
date_until_ts = datetime.strptime(
str(self.date_until), "%Y-%m-%d"
).timestamp()

try:
for entry in os.scandir(research_base):
if entry.is_dir(follow_symlinks=False):
mtime = entry.stat(follow_symlinks=False).st_mtime
if date_from_ts <= mtime <= date_until_ts:
self.services[entry.name] = {
key: value
for key, value in dictionary_template.items()
}
self.services[entry.name]["found_in_system"] = True
self.services[entry.name]["delivery_date"] = (
datetime.fromtimestamp(mtime).strftime("%Y-%m-%d")
)
except FileNotFoundError:
stderr.print(f"[red]Directory not found: {research_base}")
log.error(f"Research base directory not found: {research_base}")
raise

log.info(
f"Research folders found in time interval: {len(self.services)}"
)
raise
else:
try:
for service in rest_api.get_request(
request_info="services",
safe=True,
state="delivered",
date_from=str(self.date_from),
date_until=str(self.date_until),
):
self.services[service["service_request_number"]] = {
key: value for key, value in dictionary_template.items()
}
self.services[service["service_request_number"]][
"found_in_system"
] = True
self.services[service["service_request_number"]][
"delivery_date"
] = service["service_delivered_date"]

log.info(
f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}"
)
log.info(
"Names of the services found in said interval:"
f"{','.join([service for service in self.services.keys()])}"
)
stderr.print(
f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}"
)
stderr.print(
"Names of the services found in said interval:"
f"{','.join([service for service in self.services.keys()])}"
)
except TypeError:
stderr.print(
"Could not connect to the API (wrong password?)", style="red"
)
log.error(
"ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection."
)
raise

for service in self.services.keys():
stderr.print(service)
Expand Down Expand Up @@ -301,12 +347,12 @@ def __init__(
self.services[service]["archived_path"] = os.path.join(
conf.get_configuration("global")["archived_path"],
self.ser_type,
service_id,
service,
)
self.services[service]["non_archived_path"] = os.path.join(
conf.get_configuration("global")["data_path"],
self.ser_type,
service_id,
service,
)

self.services[service]["found"] = []
Expand Down Expand Up @@ -966,15 +1012,15 @@ def delete_non_archived_dirs(self):
for service in self.services.keys():
if not os.path.exists(self.services[service]["non_archived_path"]):
stderr.print(
f"Service {self.services[service]['non_archived_path'].split('/')[-1]}"
"has already been removed from"
f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}."
f"Service {self.services[service]['non_archived_path'].split('/')[-1]} "
"has already been removed from "
f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. "
"Nothing to delete so skipping.\n"
)
log.info(
f"Service {self.services[service]['non_archived_path'].split('/')[-1]} "
"has already been removed from"
f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}."
f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. "
"Nothing to delete so skipping.\n"
)
continue
Expand All @@ -990,14 +1036,23 @@ def delete_non_archived_dirs(self):
)
else:
stderr.print(
f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}."
f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. "
"It is safe to delete this non_archived service. Deleting.\n"
)
log.info(
f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}."
f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. "
"It is safe to delete this non_archived service. Deleting.\n"
)
shutil.rmtree(self.services[service]["non_archived_path"])
try:
shutil.rmtree(self.services[service]["non_archived_path"])
except Exception as e:
stderr.print(
f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n"
)
log.error(
f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n"
)
continue
return

def generate_tsv_table(self, filename):
Expand Down
6 changes: 3 additions & 3 deletions buisciii/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ def uncompress_targz_directory(tar_name, directory):
Untar GZ file
"""
with tarfile.open(tar_name) as out_tar:
out_tar.extractall("/".join(directory.split("/")[:-1]))
out_tar.extractall("/".join(directory.split("/")[:-1]), filter="fully_trusted")
return


Expand Down Expand Up @@ -510,15 +510,15 @@ def remake_permissions(copied_folder_path, permissions_config):
# Change permissions for directories
if "directory_chmod" in permissions_config:
subprocess.run(
f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;",
f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \\;",
shell=True,
check=True,
)

# Change permissions for files
if "file_chmod" in permissions_config:
subprocess.run(
f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;",
f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \\;",
shell=True,
check=True,
)
Loading