Merge pull request #2345 from biocore/fix-ebi-1012

fix calls to system_call and ebi submissions

Author: josenavas

qiita_pet/handlers/study_handlers/ebi_handlers.py

@@ -124,8 +124,8 @@ def post(self, preprocessed_data_id):
         study = Artifact(preprocessed_data_id).study
         state = study.ebi_submission_status
         if state == 'submitting':
-            level = 'danger'
             message = "Cannot resubmit! Current state is: %s" % state
+            self.display_template(preprocessed_data_id, message, 'danger')
         else:
             qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
             cmd = qiita_plugin.get_command('submit_to_EBI')

qiita_pet/support_files/doc/source/tutorials/prepare-information-files.rst

@@ -177,13 +177,13 @@ Without this columns you will not be able to submit to EBI. These are the column
 
 Valid values for instrument_model per platform, taken from ftp://ftp.sra.ebi.ac.uk/meta/xsd/sra_1_5/SRA.common.xsd
 
-+--------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| Platform     | Valid instrument_model options                                                                                                                                                                                                                                                                   |
-+==============+==================================================================================================================================================================================================================================================================================================+
-| ``LS454``    | ``454 GS``, ``454 GS 20``, ``454 GS FLX``, ``454 GS FLX+``, ``454 GS FLX Titanium``, ``454 GS Junior``, or ``unspecified``                                                                                                                                                                       |
-+--------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
++--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Platform     | Valid instrument_model options                                                                                                                                                                                                                                                                    |
++==============+===================================================================================================================================================================================================================================================================================================+
+| ``LS454``    | ``454 GS``, ``454 GS 20``, ``454 GS FLX``, ``454 GS FLX+``, ``454 GS FLX Titanium``, ``454 GS Junior``, or ``unspecified``                                                                                                                                                                        |
++--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``Illumina`` | ``Illumina Genome Analyzer``, ``Illumina Genome Analyzer II``, ``Illumina Genome Analyzer IIx``, ``Illumina HiSeq 2500``, ``Illumina HiSeq 2000``, ``Illumina HiSeq 1500``, ``Illumina HiSeq 1000``, ``Illumina MiSeq``, ``Illumina HiScanSQ``, ``HiSeq X Ten``, ``NextSeq 500``, ``unspecified`` |
-+--------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
++--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
 .. _required-fields-for-preprocessing-target-gene-data:
 

qiita_pet/test/test_download.py

@@ -140,7 +140,7 @@ def test_download_study(self):
             '[0-9]*.txt mapping_files/6_mapping_file.txt\n'
             '- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-'
             '[0-9]*.txt mapping_files/7_mapping_file.txt\n'
-            '- 39752 /protected/BIOM/{0}/otu_table.biom '
+            '- [0-9]* /protected/BIOM/{0}/otu_table.biom '
             'BIOM/{0}/otu_table.biom\n'
             '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
             'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'
@@ -161,7 +161,7 @@ def test_download_study(self):
         response = self.get('/download_study_bioms/1')
         self.assertEqual(response.code, 200)
         exp = (
-            '- 39752 /protected/BIOM/{0}/otu_table.biom '
+            '- [0-9]* /protected/BIOM/{0}/otu_table.biom '
             'BIOM/{0}/otu_table.biom\n'
             '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log '
             'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n'

qiita_ware/commands.py

@@ -61,26 +61,15 @@ def submit_EBI(preprocessed_data_id, action, send):
             LogEntry.create('Runtime',
                             ("Submitting sequences for pre_processed_id: "
                              "%d" % preprocessed_data_id))
-            try:
-                for cmd in ebi_submission.generate_send_sequences_cmd():
-                    try:
-                        stdout, stderr, _ = system_call(cmd)
-                    except Exception as e:
-                        stdout = ''
-                        stderr = str(e)
-                        le = LogEntry.create(
-                            'Fatal', "Command: %s\nError: %s\n" % (cmd,
-                                                                   str(e)),
-                            info={'ebi_submission': preprocessed_data_id})
-                        ebi_submission.study.ebi_submission_status = (
-                            "failed: ASCP submission, log id: %d" % le.id)
-                        raise ComputeError("EBI Submission failed! Log id: "
-                                           "%d" % le.id)
-                    finally:
-                        open(ebi_submission.ascp_reply, 'a').write(
-                            'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
-            finally:
-                environ['ASPERA_SCP_PASS'] = old_ascp_pass
+            for cmd in ebi_submission.generate_send_sequences_cmd():
+                stdout, stderr, rv = system_call(cmd)
+                if rv != 0:
+                    error_msg = ("Error:\nStd output:%s\nStd error:%s" % (
+                        stdout, stderr))
+                    raise ComputeError(error_msg)
+                open(ebi_submission.ascp_reply, 'a').write(
+                    'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
+            environ['ASPERA_SCP_PASS'] = old_ascp_pass
             LogEntry.create('Runtime',
                             ('Submission of sequences of pre_processed_id: '
                              '%d completed successfully' %
@@ -91,25 +80,18 @@ def submit_EBI(preprocessed_data_id, action, send):
         LogEntry.create('Runtime',
                         ("Submitting XMLs for pre_processed_id: "
                          "%d" % preprocessed_data_id))
-        try:
-            xml_content, stderr, _ = system_call(xmls_cmds)
-        except Exception as e:
-            xml_content = ''
-            stderr = str(e)
-            le = LogEntry.create(
-                'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)),
-                info={'ebi_submission': preprocessed_data_id})
-            ebi_submission.study.ebi_submission_status = (
-                "failed: XML submission, log id: %d" % le.id)
-            raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
+        xml_content, stderr, rv = system_call(xmls_cmds)
+        if rv != 0:
+            error_msg = ("Error:\nStd output:%s\nStd error:%s" % (
+                xml_content, stderr))
+            raise ComputeError(error_msg)
         else:
             LogEntry.create('Runtime',
                             ('Submission of sequences of pre_processed_id: '
                              '%d completed successfully' %
                              preprocessed_data_id))
-        finally:
-            open(ebi_submission.curl_reply, 'w').write(
-                'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
+        open(ebi_submission.curl_reply, 'w').write(
+            'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
 
         try:
             st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
@@ -198,7 +180,10 @@ def submit_VAMPS(artifact_id):
                                     qiita_config.vamps_pass,
                                     targz_fp,
                                     qiita_config.vamps_url))
-    obs, _, _ = system_call(cmd)
+    obs, stderr, rv = system_call(cmd)
+    if rv != 0:
+        error_msg = ("Error:\nStd output:%s\nStd error:%s" % (obs, stderr))
+        raise ComputeError(error_msg)
 
     exp = ("<html>\n<head>\n<title>Process Uploaded File</title>\n</head>\n"
            "<body>\n</body>\n</html>")

qiita_ware/private_plugin.py

@@ -97,7 +97,7 @@ def submit_to_EBI(job):
         param_vals = job.parameters.values
         artifact_id = int(param_vals['artifact'])
         submission_type = param_vals['submission_type']
-        submit_EBI(artifact_id, submission_type, False)
+        submit_EBI(artifact_id, submission_type, True)
         job._set_status('success')
 
 

scripts/qiita

@@ -61,11 +61,11 @@ def webserver(build_docs):
         sphinx_fp = join(base, 'support_files/doc/')
         cmd = 'make -C %s html' % sphinx_fp
         print('Building documentation ...')
-        try:
-            qdb.processing_job._system_call(cmd)
-        except Exception as e:
-            raise click.ClickException('Could not build documentation: %s' %
-                                       str(e))
+        stdout, stderr, rv = qdb.processing_job._system_call(cmd)
+        if rv != 0:
+            raise click.ClickException(
+                'Could not build documentation:\n'
+                'Std output:%s\nStd error:%s' % (stdout, stderr))
         else:
             print('Documentation successfully built')