Merge pull request #2 from jlab/fetch_directories

Fetch directories

Author: sjanssen2

.github/workflows/qiita-ci.yml

@@ -52,8 +52,11 @@ jobs:
 
           # we need to download qiita directly so we have "easy" access to
           # all config files
-          wget https://github.com/biocore/qiita/archive/dev.zip
-          unzip dev.zip
+          # wget https://github.com/biocore/qiita/archive/dev.zip
+          # unzip dev.zip
+          wget https://github.com/jlab/qiita/archive/refs/heads/tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip
+          unzip tornado_FetchFileFromCentralHandler_alsoDirs_debug.zip
+          mv qiita-tornado_FetchFileFromCentralHandler_alsoDirs_debug qiita-dev
 
           # pull out the port so we can modify the configuration file easily
           pgport=${{ job.services.postgres.ports[5432] }}

qiita_client/qiita_client.py

@@ -15,6 +15,9 @@
 from json import dumps
 from random import randint
 import fnmatch
+from io import BytesIO
+from zipfile import ZipFile
+
 
 try:
     from itertools import zip_longest
@@ -269,7 +272,7 @@ def _request_oauth2(self, req, rettype, *args, **kwargs):
             The request to execute
         rettype : string
             The return type of the function, either "json" or
-            if e.g. files are transferred "content"
+            "object" for the response object itself
         args : tuple
             The request args
         kwargs : dict
@@ -328,15 +331,15 @@ def _request_retry(self, req, url, rettype='json', **kwargs):
             The request to execute
         rettype : string
             The return type of the function, either "json" (default) or
-            if e.g. files are transferred "content"
+            "object" for the response object itself
         url : str
             The url to access in the server
         kwargs : dict
             The request kwargs
 
         Returns
         -------
-        dict or None or plain content IF rettype='content'
+        dict or None or response object IF rettype='object'
             The JSON information in the request response, if any
 
         Raises
@@ -391,13 +394,13 @@ def _request_retry(self, req, url, rettype='json', **kwargs):
                     if rettype is None or rettype == 'json':
                         return r.json()
                     else:
-                        if rettype == 'content':
-                            return r.content
+                        if rettype == 'object':
+                            return r
                         else:
                             raise ValueError(
                                 ("return type rettype='%s' cannot be "
                                  "understand. Choose from 'json' (default) "
-                                 "or 'content!") % rettype)
+                                 "or 'object!") % rettype)
                 except ValueError:
                     return None
             stime = randint(MIN_TIME_SLEEP, MAX_TIME_SLEEP)
@@ -418,7 +421,7 @@ def get(self, url, rettype='json', **kwargs):
             The url to access in the server
         rettype : string
             The return type of the function, either "json" (default) or
-            if e.g. files are transferred "content"
+            "object" for the response object itself
         kwargs : dict
             The request kwargs
 
@@ -746,8 +749,8 @@ def _process_files_per_sample_fastq(self, files, prep_info,
         return sample_names, prep_info
 
     def fetch_file_from_central(self, filepath, prefix=None):
-        """Moves content of a file from Qiita's central BASE_DATA_DIR to a
-           local plugin file-system.
+        """Moves content of a file or directory from Qiita's central
+           BASE_DATA_DIR to a local plugin file-system.
 
            By default, this is exactly the same location, i.e. the return
            filepath is identical to the requested one and nothing is moved /
@@ -760,22 +763,24 @@ def fetch_file_from_central(self, filepath, prefix=None):
         ----------
         filepath : str
             The filepath in Qiita's central BASE_DATA_DIR to the requested
-            file content
+            file or directory content
         prefix : str
             Primarily for testing: prefix the target filepath with this
             filepath prefix to
-            a) in 'filesystem' mode: create an actual file copy (for testing)
+            a) in 'filesystem' mode: create an actual file/directiry copy
+               (for testing)
                If prefix=None, nothing will be copied/moved
-            b) in 'https' mode: flexibility to locate files differently in
-               plugin local file system.
+            b) in 'https' mode: flexibility to locate files/directories
+               differently in plugin local file system.
 
         Returns
         -------
-        str : the filepath of the requested file within the local file system
+        str : the filepath of the requested file or directory within the local
+              file system
         """
         target_filepath = filepath
         logger.debug(
-            'Fetching file "%s" via protocol=%s from Qiita main.' % (
+            'Fetching file/directory "%s" via protocol=%s from Qiita main.' % (
                 filepath, self._plugincoupling))
 
         if (prefix is not None) and (prefix != ""):
@@ -792,7 +797,10 @@ def fetch_file_from_central(self, filepath, prefix=None):
                 if not os.path.exists(os.path.dirname(target_filepath)):
                     os.makedirs(os.path.dirname(target_filepath))
 
-                shutil.copyfile(filepath, target_filepath)
+                if os.path.isdir(filepath):
+                    shutil.copytree(filepath, target_filepath)
+                else:
+                    shutil.copyfile(filepath, target_filepath)
 
             return target_filepath
 
@@ -802,17 +810,24 @@ def fetch_file_from_central(self, filepath, prefix=None):
                 filepath = filepath[len(os.path.abspath(os.sep)):]
 
             # actual call to Qiita central to obtain file content
-            content = self.get(
+            response = self.get(
                 '/cloud/fetch_file_from_central/' + filepath,
-                rettype='content')
+                rettype='object')
+
+            # check if requested filepath is a single file OR a whole directory
+            if 'Is-Qiita-Directory' in response.headers.keys():
+                with ZipFile(BytesIO(response.content)) as zf:
+                    zf.extractall(path=target_filepath)
+            else:
+                content = response.content
 
-            # create necessary directory locally
-            if not os.path.exists(os.path.dirname(target_filepath)):
-                os.makedirs(os.path.dirname(target_filepath))
+                # create necessary directory locally
+                if not os.path.exists(os.path.dirname(target_filepath)):
+                    os.makedirs(os.path.dirname(target_filepath))
 
-            # write retrieved file content
-            with open(target_filepath, 'wb') as f:
-                f.write(content)
+                # write retrieved file content
+                with open(target_filepath, 'wb') as f:
+                    f.write(content)
 
             return target_filepath
 

qiita_client/tests/test_qiita_client.py

@@ -526,6 +526,32 @@ def test_delete_file_from_central(self):
                 # qiita main filepath, returned by delete_file_from_central
                 self.assertTrue(exists(fp_deleted))
 
+    def test_fetch_directory(self):
+        # a bit hacky, but should work as long as test database does not change
+        ainfo = self.qclient.get('/qiita_db/artifacts/1/')
+        base_data_dir = ainfo['files']['raw_forward_seqs'][0]['filepath'][
+            :(-1 * len('raw_data/1_s_G1_L001_sequences.fastq.gz'))]
+
+        # creating a LOCAL test directory within base_data_dir as the DB entry
+        # but no files exist. "job" is the according mountpoint
+        fp_test = join(base_data_dir, 'job', '2_test_folder')
+        self._create_test_dir(prefix=fp_test)
+
+        # transmitting test directory to qiita main (remote)
+        self.tester._plugincoupling = 'https'
+        self.tester.push_file_to_central(fp_test)
+        # fp_main = join(base_data_dir, join(*Path(fp_test).parts))
+
+        # fetch test directory from qiita main to a different location
+        # (=prefix) than it was generated
+        prefix = join(expanduser("~"), 'localFetch')
+        fp_obs = self.tester.fetch_file_from_central(fp_test, prefix=prefix)
+
+        # test a file of the freshly transferred directory from main has
+        # expected file content
+        with open(join(fp_obs, 'testdir', 'fileA.txt'), 'r') as f:
+            self.assertIn('contentA', '\n'.join(f.readlines()))
+
 
 if __name__ == '__main__':
     main()