diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aef13e5..92948c1f 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improved clean module so that an aux cleaning file is employed for specific services if applicable [#692](https://github.com/BU-ISCIII/buisciii-tools/pull/692). - Improved HTTP error handling in drylab_api module and fixed fix-permissions so that it does not break given insufficient permissions [#694](https://github.com/BU-ISCIII/buisciii-tools/pull/694). +- Implemented date filtering for research in archive, improved logging and error handling [#700](https://github.com/BU-ISCIII/buisciii-tools/pull/700) #### Fixes diff --git a/buisciii/archive.py b/buisciii/archive.py index 9ecd148d..ca0a3d91 100644 --- a/buisciii/archive.py +++ b/buisciii/archive.py @@ -5,6 +5,7 @@ import shutil import sys from math import pow +from datetime import datetime import rich import sysrsync @@ -159,13 +160,11 @@ def __init__( and (service_id is None) and (services_file is None) ): - if self.ser_type == "services_and_colaborations": + if self.ser_type in ["services_and_colaborations", "research"]: prompt_response = buisciii.utils.prompt_selection( "Search services by date, or by service ID?", ["Search by date", "Service ID"], ) - else: - prompt_response = "Service ID" log.info("Services chosen by: " + prompt_response) if prompt_response == "Search by date": @@ -229,37 +228,84 @@ def __init__( if (self.date_from is not None) and (self.date_until is not None): stderr.print("Asking our trusty API about selected services") log.info("Asking our trusty API about selected services") - try: - for service in rest_api.get_request( - request_info="services", - safe=True, - state="delivered", - date_from=str(self.date_from), - date_until=str(self.date_until), - ): - self.services[service["service_request_number"]] = { - key: value for key, value in dictionary_template.items() - } - self.services[service["service_request_number"]][ - "found_in_system" - ] = True - self.services[service["service_request_number"]][ - "delivery_date" - ] = service["service_delivered_date"] - - log.info(f"Services found in the time interval: {len(self.services)}") - log.info( - "Names of the services found in said interval:" - f"{','.join([service for service in self.services.keys()])}" - ) - except TypeError: - stderr.print( - "Could not connect to the API (wrong password?)", style="red" + + if self.ser_type == "research": + research_base = os.path.join( + conf.get_configuration("global")["data_path"], + self.ser_type, ) - log.error( - "ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection." + stderr.print(f"Scanning directory: {research_base}") + log.info(f"Scanning research directory: {research_base}") + + date_from_ts = datetime.strptime( + str(self.date_from), "%Y-%m-%d" + ).timestamp() + date_until_ts = datetime.strptime( + str(self.date_until), "%Y-%m-%d" + ).timestamp() + + try: + for entry in os.scandir(research_base): + if entry.is_dir(follow_symlinks=False): + mtime = entry.stat(follow_symlinks=False).st_mtime + if date_from_ts <= mtime <= date_until_ts: + self.services[entry.name] = { + key: value + for key, value in dictionary_template.items() + } + self.services[entry.name]["found_in_system"] = True + self.services[entry.name]["delivery_date"] = ( + datetime.fromtimestamp(mtime).strftime("%Y-%m-%d") + ) + except FileNotFoundError: + stderr.print(f"[red]Directory not found: {research_base}") + log.error(f"Research base directory not found: {research_base}") + raise + + log.info( + f"Research folders found in time interval: {len(self.services)}" ) - raise + else: + try: + for service in rest_api.get_request( + request_info="services", + safe=True, + state="delivered", + date_from=str(self.date_from), + date_until=str(self.date_until), + ): + self.services[service["service_request_number"]] = { + key: value for key, value in dictionary_template.items() + } + self.services[service["service_request_number"]][ + "found_in_system" + ] = True + self.services[service["service_request_number"]][ + "delivery_date" + ] = service["service_delivered_date"] + + log.info( + f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}" + ) + log.info( + "Names of the services found in said interval:" + f"{','.join([service for service in self.services.keys()])}" + ) + stderr.print( + f"Services found in the time interval ({self.date_from} - {self.date_until}): {len(self.services)}" + ) + stderr.print( + "Names of the services found in said interval:" + f"{','.join([service for service in self.services.keys()])}" + ) + except TypeError: + stderr.print( + "Could not connect to the API (wrong password?)", style="red" + ) + log.error( + "ERROR: Connection to the API was not successful. Possible reasons: wrong API password or bad connection." + ) + raise for service in self.services.keys(): stderr.print(service) @@ -301,12 +347,12 @@ def __init__( self.services[service]["archived_path"] = os.path.join( conf.get_configuration("global")["archived_path"], self.ser_type, - service_id, + service, ) self.services[service]["non_archived_path"] = os.path.join( conf.get_configuration("global")["data_path"], self.ser_type, - service_id, + service, ) self.services[service]["found"] = [] @@ -966,15 +1012,15 @@ def delete_non_archived_dirs(self): for service in self.services.keys(): if not os.path.exists(self.services[service]["non_archived_path"]): stderr.print( - f"Service {self.services[service]['non_archived_path'].split('/')[-1]}" - "has already been removed from" - f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}." + f"Service {self.services[service]['non_archived_path'].split('/')[-1]} " + "has already been removed from " + f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. " "Nothing to delete so skipping.\n" ) log.info( f"Service {self.services[service]['non_archived_path'].split('/')[-1]} " "has already been removed from" - f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])[:-1]}." + f"{'/'.join(self.services[service]['non_archived_path'].split('/')[:-1])}. " "Nothing to delete so skipping.\n" ) continue @@ -990,14 +1036,23 @@ def delete_non_archived_dirs(self): ) else: stderr.print( - f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}." + f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. " "It is safe to delete this non_archived service. Deleting.\n" ) log.info( - f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}." + f"Found archived path for service {self.services[service]['archived_path'].split('/')[-1]}. " "It is safe to delete this non_archived service. Deleting.\n" ) - shutil.rmtree(self.services[service]["non_archived_path"]) + try: + shutil.rmtree(self.services[service]["non_archived_path"]) + except Exception as e: + stderr.print( + f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n" + ) + log.error( + f"Could not delete {self.services[service]['non_archived_path']}: {e}. Skipping.\n" + ) + continue return def generate_tsv_table(self, filename): diff --git a/buisciii/utils.py b/buisciii/utils.py index 439ae507..adbcecdf 100755 --- a/buisciii/utils.py +++ b/buisciii/utils.py @@ -315,7 +315,7 @@ def uncompress_targz_directory(tar_name, directory): Untar GZ file """ with tarfile.open(tar_name) as out_tar: - out_tar.extractall("/".join(directory.split("/")[:-1])) + out_tar.extractall("/".join(directory.split("/")[:-1]), filter="fully_trusted") return @@ -510,7 +510,7 @@ def remake_permissions(copied_folder_path, permissions_config): # Change permissions for directories if "directory_chmod" in permissions_config: subprocess.run( - f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", + f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \\;", shell=True, check=True, ) @@ -518,7 +518,7 @@ def remake_permissions(copied_folder_path, permissions_config): # Change permissions for files if "file_chmod" in permissions_config: subprocess.run( - f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", + f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \\;", shell=True, check=True, )