From fdb28ba177bdbea64b7306ab7a54c7154e753a72 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 22 Jun 2026 13:52:07 +0200 Subject: [PATCH 1/4] Fixed archive module for propper logging and filtering in research --- buisciii/archive.py | 126 +++++++++++++++++++++++++++++--------------- 1 file changed, 84 insertions(+), 42 deletions(-) diff --git a/buisciii/archive.py b/buisciii/archive.py index 9ecd148d..3850edfa 100644 --- a/buisciii/archive.py +++ b/buisciii/archive.py @@ -5,6 +5,7 @@ import shutil import sys from math import pow +from datetime import datetime import rich import sysrsync @@ -159,13 +160,11 @@ def __init__( and (service_id is None) and (services_file is None) ): - if self.ser_type == "services_and_colaborations": + if self.ser_type in ["services_and_colaborations", "research"]: prompt_response = buisciii.utils.prompt_selection( "Search services by date, or by service ID?", ["Search by date", "Service ID"], ) - else: - prompt_response = "Service ID" log.info("Services chosen by: " + prompt_response) if prompt_response == "Search by date": @@ -229,37 +228,71 @@ def __init__( if (self.date_from is not None) and (self.date_until is not None): stderr.print("Asking our trusty API about selected services") log.info("Asking our trusty API about selected services") - try: - for service in rest_api.get_request( - request_info="services", - safe=True, - state="delivered", - date_from=str(self.date_from), - date_until=str(self.date_until), - ): - self.services[service["service_request_number"]] = { - key: value for key, value in dictionary_template.items() - } - self.services[service["service_request_number"]][ - "found_in_system" - ] = True - self.services[service["service_request_number"]][ - "delivery_date" - ] = service["service_delivered_date"] - - log.info(f"Services found in the time interval: {len(self.services)}") - log.info( - "Names of the services found in said interval:" - f"{','.join([service for service in self.services.keys()])}" - ) - except TypeError: - stderr.print( - "Could not connect to the API (wrong password?)", style="red" - ) - log.error( - "ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection." + + if self.ser_type == "research": + research_base = os.path.join( + conf.get_configuration("global")["data_path"], + self.ser_type, ) - raise + stderr.print(f"Scanning directory: {research_base}") + log.info(f"Scanning research directory: {research_base}") + + date_from_ts = datetime.strptime(str(self.date_from), "%Y-%m-%d").timestamp() + date_until_ts = datetime.strptime(str(self.date_until), "%Y-%m-%d").timestamp() + + try: + for entry in os.scandir(research_base): + if entry.is_dir(follow_symlinks=False): + mtime = entry.stat(follow_symlinks=False).st_mtime + if date_from_ts <= mtime <= date_until_ts: + self.services[entry.name] = { + key: value for key, value in dictionary_template.items() + } + self.services[entry.name]["found_in_system"] = True + self.services[entry.name]["delivery_date"] = datetime.fromtimestamp(mtime).strftime("%Y-%m-%d") + except FileNotFoundError: + stderr.print(f"[red]Directory not found: {research_base}") + log.error(f"Research base directory not found: {research_base}") + raise + + log.info(f"Research folders found in time interval: {len(self.services)}") + else: + try: + for service in rest_api.get_request( + request_info="services", + safe=True, + state="delivered", + date_from=str(self.date_from), + date_until=str(self.date_until), + ): + self.services[service["service_request_number"]] = { + key: value for key, value in dictionary_template.items() + } + self.services[service["service_request_number"]][ + "found_in_system" + ] = True + self.services[service["service_request_number"]][ + "delivery_date" + ] = service["service_delivered_date"] + + log.info(f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}") + log.info( + "Names of the services found in said interval:" + f"{','.join([service for service in self.services.keys()])}" + ) + stderr.print(f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}") + stderr.print( + "Names of the services found in said interval:" + f"{','.join([service for service in self.services.keys()])}" + ) + except TypeError: + stderr.print( + "Could not connect to the API (wrong password?)", style="red" + ) + log.error( + "ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection." + ) + raise for service in self.services.keys(): stderr.print(service) @@ -301,12 +334,12 @@ def __init__( self.services[service]["archived_path"] = os.path.join( conf.get_configuration("global")["archived_path"], self.ser_type, - service_id, + service, ) self.services[service]["non_archived_path"] = os.path.join( conf.get_configuration("global")["data_path"], self.ser_type, - service_id, + service, ) self.services[service]["found"] = [] @@ -966,15 +999,15 @@ def delete_non_archived_dirs(self): for service in self.services.keys(): if not os.path.exists(self.services[service]["non_archived_path"]): stderr.print( - f"Service {self.services[service]['non_archived_path'].split('/')[-1]}" - "has already been removed from" - f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}." + f"Service {self.services[service]['non_archived_path'].split('/')[-1]} " + "has already been removed from " + f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. " "Nothing to delete so skipping.\n" ) log.info( f"Service {self.services[service]['non_archived_path'].split('/')[-1]} " "has already been removed from" - f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}." + f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. " "Nothing to delete so skipping.\n" ) continue @@ -990,14 +1023,23 @@ def delete_non_archived_dirs(self): ) else: stderr.print( - f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}." + f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. " "It is safe to delete this non_archived service. Deleting.\n" ) log.info( - f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}." + f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. " "It is safe to delete this non_archived service. Deleting.\n" ) - shutil.rmtree(self.services[service]["non_archived_path"]) + try: + shutil.rmtree(self.services[service]["non_archived_path"]) + except Exception as e: + stderr.print( + f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n" + ) + log.error( + f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n" + ) + continue return def generate_tsv_table(self, filename): From d812fff634c703a279217b468733515ddeffd079 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 22 Jun 2026 13:53:20 +0200 Subject: [PATCH 2/4] Fixed archive unzip in utils --- buisciii/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buisciii/utils.py b/buisciii/utils.py index 439ae507..adbcecdf 100755 --- a/buisciii/utils.py +++ b/buisciii/utils.py @@ -315,7 +315,7 @@ def uncompress_targz_directory(tar_name, directory): Untar GZ file """ with tarfile.open(tar_name) as out_tar: - out_tar.extractall("/".join(directory.split("/")[:-1])) + out_tar.extractall("/".join(directory.split("/")[:-1]), filter="fully_trusted") return @@ -510,7 +510,7 @@ def remake_permissions(copied_folder_path, permissions_config): # Change permissions for directories if "directory_chmod" in permissions_config: subprocess.run( - f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", + f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \\;", shell=True, check=True, ) @@ -518,7 +518,7 @@ def remake_permissions(copied_folder_path, permissions_config): # Change permissions for files if "file_chmod" in permissions_config: subprocess.run( - f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", + f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \\;", shell=True, check=True, ) From 3800562d607daaa91c7a62df700738dd555ea8d2 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 22 Jun 2026 14:04:46 +0200 Subject: [PATCH 3/4] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aef13e5..92948c1f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improved clean module so that an aux cleaning file is employed for specific services if applicable [#692](https://github.com/BU-ISCIII/buisciii-tools/pull/692). - Improved HTTP error handling in drylab_api module and fixed fix-permissions so that it does not break given insufficient permissions [#694](https://github.com/BU-ISCIII/buisciii-tools/pull/694). +- Implemented date filtering for research in archive, improved logging and error handling [#700](https://github.com/BU-ISCIII/buisciii-tools/pull/700) #### Fixes From edcb648da4035a5b0209209cd5a18b89036843e7 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 22 Jun 2026 14:09:35 +0200 Subject: [PATCH 4/4] black --- buisciii/archive.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/buisciii/archive.py b/buisciii/archive.py index 3850edfa..ca0a3d91 100644 --- a/buisciii/archive.py +++ b/buisciii/archive.py @@ -228,7 +228,7 @@ def __init__( if (self.date_from is not None) and (self.date_until is not None): stderr.print("Asking our trusty API about selected services") log.info("Asking our trusty API about selected services") - + if self.ser_type == "research": research_base = os.path.join( conf.get_configuration("global")["data_path"], @@ -236,26 +236,35 @@ def __init__( ) stderr.print(f"Scanning directory: {research_base}") log.info(f"Scanning research directory: {research_base}") - - date_from_ts = datetime.strptime(str(self.date_from), "%Y-%m-%d").timestamp() - date_until_ts = datetime.strptime(str(self.date_until), "%Y-%m-%d").timestamp() - + + date_from_ts = datetime.strptime( + str(self.date_from), "%Y-%m-%d" + ).timestamp() + date_until_ts = datetime.strptime( + str(self.date_until), "%Y-%m-%d" + ).timestamp() + try: for entry in os.scandir(research_base): if entry.is_dir(follow_symlinks=False): mtime = entry.stat(follow_symlinks=False).st_mtime if date_from_ts <= mtime <= date_until_ts: self.services[entry.name] = { - key: value for key, value in dictionary_template.items() + key: value + for key, value in dictionary_template.items() } self.services[entry.name]["found_in_system"] = True - self.services[entry.name]["delivery_date"] = datetime.fromtimestamp(mtime).strftime("%Y-%m-%d") + self.services[entry.name]["delivery_date"] = ( + datetime.fromtimestamp(mtime).strftime("%Y-%m-%d") + ) except FileNotFoundError: stderr.print(f"[red]Directory not found: {research_base}") log.error(f"Research base directory not found: {research_base}") raise - - log.info(f"Research folders found in time interval: {len(self.services)}") + + log.info( + f"Research folders found in time interval: {len(self.services)}" + ) else: try: for service in rest_api.get_request( @@ -275,12 +284,16 @@ def __init__( "delivery_date" ] = service["service_delivered_date"] - log.info(f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}") + log.info( + f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}" + ) log.info( "Names of the services found in said interval:" f"{','.join([service for service in self.services.keys()])}" ) - stderr.print(f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}") + stderr.print( + f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}" + ) stderr.print( "Names of the services found in said interval:" f"{','.join([service for service in self.services.keys()])}"