Merge pull request #10 from jlab/auth_oidc_merged

Auth OIDC merged

Author: sjanssen2

.github/workflows/qiita-ci.yml

@@ -104,9 +104,7 @@ jobs:
       - name: Install plugins
         shell: bash -l {0}
         run: |
-          wget https://data.qiime2.org/distro/core/qiime2-2022.11-py38-linux-conda.yml
-          conda env create --quiet -n qtp-biom --file qiime2-2022.11-py38-linux-conda.yml
-          rm qiime2-2022.11-py38-linux-conda.yml
+          conda env create -n qtp-biom --file https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml
           export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
           export REDBIOM_HOST="http://localhost:7379"
@@ -181,7 +179,7 @@ jobs:
           echo "Connecting as $USER@localhost"
           # this line (and the -o StrictHostKeyChecking=no) is so the server
           # is added to the list of known servers
-          scp -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
+          scp -O -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
 
       - name: Main tests
         shell: bash -l {0}

CHANGELOG.md

@@ -1,5 +1,72 @@
 # Qiita changelog
 
+Version 2025.02
+---------------
+
+Deployed on February 24th, 2025
+
+* Replaced os.rename for shutil.move in the code to fix [#3455](https://github.com/qiita-spots/qiita/issues/3455).
+* Via qp-spades, replaced the legacy `spades` command for `cloudSPAdes` for TellSeq.
+* `FASTA_preprocessed` within qtp-sequencing now allows for results to be named using their sample-name, extra from run-prefix.
+* `Remove SynDNA inserts & plasmid reads` superseded `Remove SynDNA reads`, which now removes SynDna inserts and plasmids.
+* `update_resource_allocation_redis` now relies on using equations stored in the database vs. hardcoded; thank you @Gossty!
+* SPP: Updated prep-info file generation to identify and report filtered fastq files that could not be matched to a sample-id instead of silently ignoring them.
+* SPP: Removed legacy test code and example files for amplicon processing. Some other tests updated and repurposed.
+* SPP: jobs are now easier to restart.
+* SPP: MultiQC report generation is now a separate slurm job & use jinja2 templates; also FastQC use jinja2 templates.
+
+
+Version 2025.01
+---------------
+
+Deployed on January 15th, 2025
+
+* The Analysis owner is now displayed in the analysis list and the individual analysis page.
+* Admins can now use the per-preparation "Download Data Release" button to get a "BIOM" release; this version is focus on NPH data releases.
+* Improved complete_job creation time, which should result in Qiita jobs ([multiple steps](https://qiita.ucsd.edu/static/doc/html/dev/resource_allocation.html) finishing faster; for bencharks visit [patch 93.sql](https://github.com/qiita-spots/qiita/blob/master/qiita_db/support_files/patches/93.sql).
+* SPP improvements: TellSeq support added; plugin refactored to allow for easier additions like TellSeq in the future. Job restart greatly improved. Much improved handling of sample-names and ids that contain substrings like ‘I1’ and ‘R2’. New SequenceCount job can count sequences and base-pairs in parallel for any list of fastq files.
+* Other general fixes [#3440](https://github.com/qiita-spots/qiita/pull/3440), [#3445](https://github.com/qiita-spots/qiita/pull/3445), [#3446](https://github.com/qiita-spots/qiita/pull/3446),
+
+
+Version 2024.10
+---------------
+
+Deployed on October 14th, 2024
+
+* Added update_resource_allocation_redis and companion code, so resource allocations summaries are available for review. Thank you @Gossty!
+* Now is possible to have default workflows with only one step.
+* `qiita_client.update_job_step` now accepts an ignore_error optional parameter. Thank you @charles-cowart!
+* Initial changes in `qiita_client` to have more accurate variable names: `QIITA_SERVER_CERT` -> `QIITA_ROOTCA_CERT`. Thank you @charles-cowart!
+* Added `get_artifact_html_summary` to `qiita_client` to retrieve the summary file of an artifact.
+* Re-added github actions to `https://github.com/qiita-spots/qiita_client`.
+* `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support.
+* `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates.
+* `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks!
+* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440).
+* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225).
+* We updated the available parameters for `Filter features against reference [filter_features]`, `Non V4 16S sequence assessment [non_v4_16s]` and all the phylogenetic analytical commands so they can use `Greengenes2 2024.09`.
+
+
+
+Version 2024.07
+---------------
+
+Deployed on July 15th, 2024
+
+* On June 14th, 2024 we modified the SPP to use ["fastp & minimap2 against GRCh38.p14 + Phi X 174 + T2T-CHM13v2.0, then Movi against GRCh38.p14, T2T-CHM13v2.0 + Human Pangenome Reference Consortium release 2023"](https://github.com/cguccione/human_host_filtration) to filter human-reads.
+* Full refactor of the [DB patching system](https://github.com/qiita-spots/qiita/blob/master/CONTRIBUTING.md#patch-91sql) to make sure that a new production deployment has a fully empty database.
+* Fully removed Qiimp from Qiita.
+* Users can now add `ORCID`, `ResearchGate` and/or `GoogleScholar` information to their profile and the creation (registration) timestamp is kept in the database. Thank you @jlab.
+* Admins can now track and purge non-confirmed users from the database via the GUI (`/admin/purge_users/`). Thank you @jlab.
+* Added `qiita.slurm_resource_allocations` to store general job resource usage, which can be populated by `qiita_db.util.update_resource_allocation_table`.
+* Added `qiita_db.util.resource_allocation_plot` to generate different models to allocate resources from a given software command based on previous jobs, thank you @Gossty !
+* The stats page map can be centered via the configuration file; additionally, the Help and Admin emails are defined also via the configuration files, thank you @jlab !
+* ``Sequel IIe``, ``Revio``, and ``Onso`` are now valid instruments for the ``PacBio_SMRT`` platform.
+* Added `current_human_filtering` to the prep-information and `human_reads_filter_method` to the artifact to keep track of the method that it was used to human reads filter the raw artifact and know if it's up to date with what is expected via the best practices.
+* Added `reprocess_job_id` to the prep-information so we keep track if a preparation has been reprocessed with another job.
+* Other general fixes, like [#3385](https://github.com/qiita-spots/qiita/pull/3385), [#3397](https://github.com/qiita-spots/qiita/pull/3397), [#3399](https://github.com/qiita-spots/qiita/pull/3399), [#3400](https://github.com/qiita-spots/qiita/pull/3400), [#3409](https://github.com/qiita-spots/qiita/pull/3409), [#3410](https://github.com/qiita-spots/qiita/pull/3410).
+
+
 Version 2024.02
 ---------------
 
@@ -167,7 +234,7 @@ Version 2021.11
 * Allow chucked download of metadata files in analyses; this allows to process large meta-analysis (like those for The Microsetta Initiative) without worker blockage.
 * Added to the qp-qiime2 plugin the possibility of filtering tables based on system available "FeatureData[Sequence]"; to start we added 90/100/150 bps bloom tables.
 * Now we can instantiate a study via their title (Study.from_title); this will facilitate orchestration with qebil.
-* Speed up Study listing for admins and general users; the admin study display came down from 20 to 2 seconds.  
+* Speed up Study listing for admins and general users; the admin study display came down from 20 to 2 seconds.
 * Fixed the following issues: [3142](https://github.com/qiita-spots/qiita/issues/3142), [3149](https://github.com/qiita-spots/qiita/issues/3149), [3150](https://github.com/qiita-spots/qiita/issues/3150), [3119](https://github.com/qiita-spots/qiita/issues/3119), and [3160](https://github.com/qiita-spots/qiita/issues/3160).
 
 

qiita_core/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2024.02"
+__version__ = "2025.02"

qiita_db/__init__.py

@@ -27,7 +27,7 @@
 from . import user
 from . import processing_job
 
-__version__ = "2024.02"
+__version__ = "2025.02"
 
 __all__ = ["analysis", "artifact",  "archive", "base", "commands",
            "environment_manager", "exceptions", "investigation", "logger",

qiita_db/analysis.py

@@ -215,6 +215,22 @@ def create(cls, owner, name, description, from_default=False,
         job.submit()
         return instance
 
+    @classmethod
+    def delete_analysis_artifacts(cls, _id):
+        """Deletes the artifacts linked to an artifact and then the analysis
+
+        Parameters
+        ----------
+        _id : int
+            The analysis id
+        """
+        analysis = cls(_id)
+        aids = [a.id for a in analysis.artifacts if not a.parents]
+        aids.sort(reverse=True)
+        for aid in aids:
+            qdb.artifact.Artifact.delete(aid)
+        cls.delete(analysis.id)
+
     @classmethod
     def delete(cls, _id):
         """Deletes an analysis

qiita_db/archive.py

@@ -116,6 +116,7 @@ def get_merging_scheme_from_job(cls, job):
             acmd = job.command
             parent = job.input_artifacts[0]
             parent_pparameters = parent.processing_parameters
+            phms = None
             if parent_pparameters is None:
                 parent_cmd_name = None
                 parent_parameters = None
@@ -125,12 +126,26 @@ def get_merging_scheme_from_job(cls, job):
                 parent_cmd_name = pcmd.name
                 parent_parameters = parent_pparameters.values
                 parent_merging_scheme = pcmd.merging_scheme
-
-            return qdb.util.human_merging_scheme(
+                if not parent_merging_scheme['ignore_parent_command']:
+                    gp = parent.parents[0]
+                    gp_params = gp.processing_parameters
+                    if gp_params is not None:
+                        gp_cmd = gp_params.command
+                        phms = qdb.util.human_merging_scheme(
+                            parent_cmd_name, parent_merging_scheme,
+                            gp_cmd.name, gp_cmd.merging_scheme,
+                            parent_parameters, [], gp_params.values)
+
+            hms = qdb.util.human_merging_scheme(
                 acmd.name, acmd.merging_scheme,
                 parent_cmd_name, parent_merging_scheme,
                 job.parameters.values, [], parent_parameters)
 
+            if phms is not None:
+                hms = qdb.util.merge_overlapping_strings(hms, phms)
+
+            return hms
+
     @classmethod
     def retrieve_feature_values(cls, archive_merging_scheme=None,
                                 features=None):

qiita_db/artifact.py

@@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self):
             # words has more that one processing step behind it
             fine_to_send = []
             fine_to_send.extend([pt.artifact for pt in self.prep_templates])
-            fine_to_send.extend([c for a in fine_to_send for c in a.children])
+            fine_to_send.extend([c for a in fine_to_send if a is not None
+                                 for c in a.children])
             if self not in fine_to_send:
                 return False
 
@@ -1342,23 +1343,6 @@ def _helper(sql_edges, edges, nodes):
                         # If the job is in success we don't need to do anything
                         # else since it would've been added by the code above
                         if jstatus != 'success':
-                            # Connect the job with his input artifacts, the
-                            # input artifacts may or may not exist yet, so we
-                            # need to check both the input_artifacts and the
-                            # pending properties
-                            for in_art in n_obj.input_artifacts:
-                                iid = in_art.id
-                                if iid not in nodes and iid in extra_nodes:
-                                    nodes[iid] = extra_nodes[iid]
-                                _add_edge(edges, nodes[iid], nodes[n_obj.id])
-
-                            pending = n_obj.pending
-                            for pred_id in pending:
-                                for pname in pending[pred_id]:
-                                    in_node_id = '%s:%s' % (
-                                        pred_id, pending[pred_id][pname])
-                                    _add_edge(edges, nodes[in_node_id],
-                                              nodes[n_obj.id])
 
                             if jstatus != 'error':
                                 # If the job is not errored, we can add the
@@ -1380,6 +1364,34 @@ def _helper(sql_edges, edges, nodes):
                                     queue.append(cjob.id)
                                     if cjob.id not in nodes:
                                         nodes[cjob.id] = ('job', cjob)
+
+                                    # including the outputs
+                                    for o_name, o_type in cjob.command.outputs:
+                                        node_id = '%s:%s' % (cjob.id, o_name)
+                                        node = TypeNode(
+                                            id=node_id, job_id=cjob.id,
+                                            name=o_name, type=o_type)
+                                        if node_id not in nodes:
+                                            nodes[node_id] = ('type', node)
+
+                            # Connect the job with his input artifacts, the
+                            # input artifacts may or may not exist yet, so we
+                            # need to check both the input_artifacts and the
+                            # pending properties
+                            for in_art in n_obj.input_artifacts:
+                                iid = in_art.id
+                                if iid not in nodes and iid in extra_nodes:
+                                    nodes[iid] = extra_nodes[iid]
+                                _add_edge(edges, nodes[iid], nodes[n_obj.id])
+
+                            pending = n_obj.pending
+                            for pred_id in pending:
+                                for pname in pending[pred_id]:
+                                    in_node_id = '%s:%s' % (
+                                        pred_id, pending[pred_id][pname])
+                                    _add_edge(edges, nodes[in_node_id],
+                                              nodes[n_obj.id])
+
                     elif n_type == 'type':
                         # Connect this 'future artifact' with the job that will
                         # generate it
@@ -1684,3 +1696,51 @@ def get_commands(self):
                 cids = cmds & cids
 
             return [qdb.software.Command(cid) for cid in cids]
+
+    @property
+    def human_reads_filter_method(self):
+        """The human_reads_filter_method of the artifact
+
+        Returns
+        -------
+        str
+            The human_reads_filter_method name
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT human_reads_filter_method
+                     FROM qiita.artifact
+                     LEFT JOIN qiita.human_reads_filter_method
+                        USING (human_reads_filter_method_id)
+                     WHERE artifact_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [self.id])
+            return qdb.sql_connection.TRN.execute_fetchlast()
+
+    @human_reads_filter_method.setter
+    def human_reads_filter_method(self, value):
+        """Set the human_reads_filter_method of the artifact
+
+        Parameters
+        ----------
+        value : str
+            The new artifact's human_reads_filter_method
+
+        Raises
+        ------
+        ValueError
+            If `value` doesn't exist in the database
+        """
+        with qdb.sql_connection.TRN:
+            sql = """SELECT human_reads_filter_method_id
+                     FROM qiita.human_reads_filter_method
+                     WHERE human_reads_filter_method = %s"""
+            qdb.sql_connection.TRN.add(sql, [value])
+            idx = qdb.sql_connection.TRN.execute_fetchflatten()
+
+            if len(idx) == 0:
+                raise ValueError(
+                    f'"{value}" is not a valid human_reads_filter_method')
+
+            sql = """UPDATE qiita.artifact
+                     SET human_reads_filter_method_id = %s
+                     WHERE artifact_id = %s"""
+            qdb.sql_connection.TRN.add(sql, [idx[0], self.id])

qiita_db/handlers/processing_job.py

@@ -146,7 +146,9 @@ def post(self, job_id):
                 cmd, values_dict={'job_id': job_id,
                                   'payload': self.request.body.decode(
                                       'ascii')})
-            job = qdb.processing_job.ProcessingJob.create(job.user, params)
+            # complete_job are unique so it is fine to force them to be created
+            job = qdb.processing_job.ProcessingJob.create(
+                job.user, params, force=True)
             job.submit()
 
         self.finish()

qiita_db/handlers/tests/test_processing_job.py

@@ -233,9 +233,9 @@ def test_post_job_success(self):
         self.assertIsNotNone(cj)
         # additionally we can test that job.print_trace is correct
         self.assertEqual(job.trace, [
-            f'{job.id} [Not Available]: Validate | '
+            f'{job.id} [Not Available] (success): Validate | '
             '-p qiita -N 1 -n 1 --mem 90gb --time 150:00:00 --nice=10000',
-            f'  {cj.id} [{cj.external_id}] | '
+            f'  {cj.id} [{cj.external_id}] (success)| '
             '-p qiita -N 1 -n 1 --mem 16gb --time 10:00:00 --nice=10000'])
 
     def test_post_job_success_with_archive(self):