From 20c679a7b20225637573867e3c4f1e722d9c898a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Mar 2025 17:15:00 +0100 Subject: [PATCH 1/5] add function that creates neccessary "mountpoints" --- qiita_db/environment_manager.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index 95fa8c468..0b0ca8b07 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------------------------- from os.path import abspath, dirname, join, exists, basename, splitext +from shutil import copytree from functools import partial from os import mkdir import gzip @@ -126,6 +127,34 @@ def _download_reference_files(): _insert_processed_params(ref) +def create_mountpoints(): + r"""In a fresh qiita setup, sub-directories under qiita_config.base_data_dir + might not yet exist. To avoid failing in later steps, they are created + here. + """ + with qdb.sql_connection.TRN: + # Insert the settings values to the database + sql = """SELECT mountpoint FROM qiita.data_directory + WHERE active = TRUE""" + qdb.sql_connection.TRN.add(sql) + created_subdirs = [] + for subdir in qdb.sql_connection.TRN.execute_fetchflatten(): + if not exists(join(qiita_config.base_data_dir, subdir)): + if qiita_config.test_environment and \ + exists(get_support_file('test_data', subdir)): + # if in test mode, we want to potentially fill the + # new directory with according test data + copytree(get_support_file('test_data', subdir), + join(qiita_config.base_data_dir, subdir)) + else: + # in production mode, an empty directory is created + mkdir(join(qiita_config.base_data_dir, subdir)) + created_subdirs.append(subdir) + + if len(created_subdirs) > 0: + print("Created %i sub-directories as 'mount points' in '%s': %s" % ( + len(created_subdirs), qiita_config.base_data_dir, + ', '.join(created_subdirs))) def make_environment(load_ontologies, download_reference, add_demo_user): r"""Creates the new environment specified in the configuration @@ -397,6 +426,9 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False): with qdb.sql_connection.TRN: _populate_test_db() + # create mountpoints as subdirectories in BASE_DATA_DIR + create_mountpoints() + patch_update_sql = "UPDATE settings SET current_patch = %s" for sql_patch_fp in sql_patch_files[next_patch_index:]: sql_patch_filename = basename(sql_patch_fp) From 80c4b879712ed263464da22226b2b6db90173c30 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Mar 2025 17:26:07 +0100 Subject: [PATCH 2/5] code style --- qiita_db/environment_manager.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index 0b0ca8b07..936325c24 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -127,10 +127,11 @@ def _download_reference_files(): _insert_processed_params(ref) + def create_mountpoints(): - r"""In a fresh qiita setup, sub-directories under qiita_config.base_data_dir - might not yet exist. To avoid failing in later steps, they are created - here. + r"""In a fresh qiita setup, sub-directories under + qiita_config.base_data_dir might not yet exist. To avoid failing in + later steps, they are created here. """ with qdb.sql_connection.TRN: # Insert the settings values to the database @@ -152,9 +153,10 @@ def create_mountpoints(): created_subdirs.append(subdir) if len(created_subdirs) > 0: - print("Created %i sub-directories as 'mount points' in '%s': %s" % ( - len(created_subdirs), qiita_config.base_data_dir, - ', '.join(created_subdirs))) + print("Created %i sub-directories as 'mount points' in '%s': %s" + % (len(created_subdirs), qiita_config.base_data_dir, + ', '.join(created_subdirs))) + def make_environment(load_ontologies, download_reference, add_demo_user): r"""Creates the new environment specified in the configuration From 29865d768d6cfc9d029ecbcb0eed2a002e6d3350 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 6 Mar 2025 20:46:40 +0100 Subject: [PATCH 3/5] addressing Antonio's suggestion to use get_mountpoint --- qiita_db/environment_manager.py | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index 936325c24..eca54e23d 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -134,28 +134,29 @@ def create_mountpoints(): later steps, they are created here. """ with qdb.sql_connection.TRN: - # Insert the settings values to the database - sql = """SELECT mountpoint FROM qiita.data_directory + sql = """SELECT DISTINCT mountpoint FROM qiita.data_directory WHERE active = TRUE""" qdb.sql_connection.TRN.add(sql) created_subdirs = [] - for subdir in qdb.sql_connection.TRN.execute_fetchflatten(): - if not exists(join(qiita_config.base_data_dir, subdir)): - if qiita_config.test_environment and \ - exists(get_support_file('test_data', subdir)): - # if in test mode, we want to potentially fill the - # new directory with according test data - copytree(get_support_file('test_data', subdir), - join(qiita_config.base_data_dir, subdir)) - else: - # in production mode, an empty directory is created - mkdir(join(qiita_config.base_data_dir, subdir)) - created_subdirs.append(subdir) + for mountpoint in qdb.sql_connection.TRN.execute_fetchflatten(): + for (ddid, subdir) in qdb.util.get_mountpoint(mountpoint, + retrieve_all=True): + if not exists(join(qiita_config.base_data_dir, subdir)): + if qiita_config.test_environment and \ + exists(get_support_file('test_data', subdir)): + # if in test mode, we want to potentially fill the + # new directory with according test data + copytree(get_support_file('test_data', subdir), + join(qiita_config.base_data_dir, subdir)) + else: + # in production mode, an empty directory is created + mkdir(join(qiita_config.base_data_dir, subdir)) + created_subdirs.append(subdir) if len(created_subdirs) > 0: - print("Created %i sub-directories as 'mount points' in '%s': %s" - % (len(created_subdirs), qiita_config.base_data_dir, - ', '.join(created_subdirs))) + print("Created %i sub-directories as 'mount points':\n%s" + % (len(created_subdirs), + ''.join(map(lambda x: ' - %s\n' % x, created_subdirs)))) def make_environment(load_ontologies, download_reference, add_demo_user): From 1ea92a1ed0f24a9ab722b741a707890542e01db2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 9 Mar 2025 12:02:15 +0100 Subject: [PATCH 4/5] raise error if src dir is not present at expected location --- qiita_db/environment_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_db/environment_manager.py b/qiita_db/environment_manager.py index eca54e23d..f8b97aa0b 100644 --- a/qiita_db/environment_manager.py +++ b/qiita_db/environment_manager.py @@ -142,8 +142,7 @@ def create_mountpoints(): for (ddid, subdir) in qdb.util.get_mountpoint(mountpoint, retrieve_all=True): if not exists(join(qiita_config.base_data_dir, subdir)): - if qiita_config.test_environment and \ - exists(get_support_file('test_data', subdir)): + if qiita_config.test_environment: # if in test mode, we want to potentially fill the # new directory with according test data copytree(get_support_file('test_data', subdir), From 5f87771633d7809979dbe599c33a8c0b0b691f0f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 9 Mar 2025 12:15:35 +0100 Subject: [PATCH 5/5] adding mountpoint directory FASTQ --- qiita_db/support_files/test_data/FASTQ/blank.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 qiita_db/support_files/test_data/FASTQ/blank.txt diff --git a/qiita_db/support_files/test_data/FASTQ/blank.txt b/qiita_db/support_files/test_data/FASTQ/blank.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/qiita_db/support_files/test_data/FASTQ/blank.txt @@ -0,0 +1 @@ +