From b28670d7255d85cd80e8cf88b318428388df3119 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 28 Jun 2024 11:23:38 -0600 Subject: [PATCH 01/25] Clean db files add human reads filter method (#3421) * Update CHANGELOG.md * clean db files + add human_reads_filter_method * fix db errors * modify inserts to be in the correct order --- qiita_db/support_files/patches/92.sql | 25 ++ .../support_files/patches/test_db_sql/92.sql | 13 +- qiita_db/support_files/populate_test_db.sql | 10 +- qiita_db/support_files/qiita-db-unpatched.sql | 5 +- qiita_db/support_files/qiita-db.dbs | 35 ++- qiita_db/support_files/qiita-db.html | 257 ++++++++++++------ 6 files changed, 244 insertions(+), 101 deletions(-) diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index 19d284b36..b618c3133 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -39,3 +39,28 @@ ALTER TABLE qiita.qiita_user ADD creation_timestamp timestamp without time zone DEFAULT NOW(); COMMENT ON COLUMN qiita.qiita_user.creation_timestamp IS 'The date the user account was created'; + +-- Jun 28, 2024 +-- These columns were added by mistake to qiita-db-unpatched.sql in PR: +-- https://github.com/qiita-spots/qiita/pull/3412 so adding here now + +ALTER TABLE qiita.qiita_user ADD social_orcid character varying DEFAULT NULL; +ALTER TABLE qiita.qiita_user ADD social_researchgate character varying DEFAULT NULL; +ALTER TABLE qiita.qiita_user ADD social_googlescholar character varying DEFAULT NULL; + +-- Add human_reads_filter_method so we can keep track of the available methods +-- and link them to the preparations + +CREATE TABLE qiita.human_reads_filter_method ( + human_reads_filter_method_id bigint NOT NULL, + human_reads_filter_method_method character varying NOT NULL, + CONSTRAINT pk_human_reads_filter_method_id PRIMARY KEY ( + human_reads_filter_method_id ) + ); + +ALTER TABLE qiita.prep_template + ADD human_reads_filter_method_id bigint DEFAULT NULL; +ALTER TABLE qiita.prep_template + ADD CONSTRAINT fk_human_reads_filter_method + FOREIGN KEY ( human_reads_filter_method_id ) + REFERENCES qiita.human_reads_filter_method ( human_reads_filter_method_id ); diff --git a/qiita_db/support_files/patches/test_db_sql/92.sql b/qiita_db/support_files/patches/test_db_sql/92.sql index 65266e9f0..8346b18e1 100644 --- a/qiita_db/support_files/patches/test_db_sql/92.sql +++ b/qiita_db/support_files/patches/test_db_sql/92.sql @@ -929,11 +929,16 @@ INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, columns -- for testing: provide creation date for one of the existing users -UPDATE qiita.qiita_user SET creation_timestamp = '2015-12-03 13:52:42.751331-07' WHERE email = 'test@foo.bar'; +UPDATE qiita.qiita_user SET + social_orcid = '0000-0002-0975-9019', + social_researchgate = 'Rob-Knight', + social_googlescholar = '_e3QL94AAAAJ', + creation_timestamp = '2015-12-03 13:52:42.751331-07' +WHERE email = 'test@foo.bar'; -- Jun 20, 2024 -- Add some non-verified users to the test DB to test new admin page: /admin/purge_users/ -INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW()); -INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '1 YEAR'); -INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NULL, NULL, NULL, NOW() - INTERVAL '30 DAY'); +INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NOW(), NULL, NULL, NULL); +INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NOW() - INTERVAL '1 YEAR', NULL, NULL, NULL); +INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NOW() - INTERVAL '30 DAY', NULL, NULL, NULL); diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index 1ca2edb66..ea7c4337c 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -50,10 +50,10 @@ INSERT INTO qiita.user_level VALUES (7, 'wet-lab admin', 'Can access the private -- Data for Name: qiita_user; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false, '0000-0002-0975-9019', 'Rob-Knight', '_e3QL94AAAAJ'); -INSERT INTO qiita.qiita_user VALUES ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL, false); -INSERT INTO qiita.qiita_user VALUES ('admin@foo.bar', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL, false); -INSERT INTO qiita.qiita_user VALUES ('demo@microbio.me', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL, false); +INSERT INTO qiita.qiita_user VALUES ('test@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL); +INSERT INTO qiita.qiita_user VALUES ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344', NULL, NULL, NULL); +INSERT INTO qiita.qiita_user VALUES ('admin@foo.bar', 1, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Admin', 'Owner University', '312 noname st, Apt K, Nonexistantown, CO 80302', '222-444-6789', NULL, NULL, NULL); +INSERT INTO qiita.qiita_user VALUES ('demo@microbio.me', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Demo', 'Qiita Dev', '1345 Colorado Avenue', '303-492-1984', NULL, NULL, NULL); -- @@ -329,7 +329,7 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin -- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog -- -INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); +INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757); INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546); INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377); INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986); diff --git a/qiita_db/support_files/qiita-db-unpatched.sql b/qiita_db/support_files/qiita-db-unpatched.sql index a61b4645d..1ce86de39 100644 --- a/qiita_db/support_files/qiita-db-unpatched.sql +++ b/qiita_db/support_files/qiita-db-unpatched.sql @@ -1888,10 +1888,7 @@ CREATE TABLE qiita.qiita_user ( user_verify_code character varying, pass_reset_code character varying, pass_reset_timestamp timestamp without time zone, - receive_processing_job_emails boolean DEFAULT false, - social_orcid character varying DEFAULT NULL, - social_researchgate character varying DEFAULT NULL, - social_googlescholar character varying DEFAULT NULL + receive_processing_job_emails boolean DEFAULT false ); diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index d2cb68131..cf84177af 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -735,6 +735,13 @@ + + + + + + +
@@ -1007,6 +1014,9 @@ + + + @@ -1265,6 +1275,19 @@ + + + + + + + + + + + + + @@ -1370,6 +1393,13 @@ + + + + + + + @@ -2098,6 +2128,7 @@ $function$ + @@ -2125,14 +2156,14 @@ $function$ - + - + diff --git a/qiita_db/support_files/qiita-db.html b/qiita_db/support_files/qiita-db.html index 38d3fe23b..80fb3a211 100644 --- a/qiita_db/support_files/qiita-db.html +++ b/qiita_db/support_files/qiita-db.html @@ -2,7 +2,7 @@ -Qiita Db +QiitaDB @@ -291,6 +291,11 @@ + + + + + @@ -299,7 +304,7 @@ - Qiita Db + QiitaDB (c) DbSchema Hover columns to read the comments. @@ -410,11 +415,11 @@ analysis_users ref analysis ( analysis_id ) analysis_id - + Fk fk_analysis_users_user analysis_users ref qiita_user ( email ) -Fk fk_analysis_users_user +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 368,256L 368,1224Q 368,1232 360,1232L 336,1232' ><title>Fk fk_analysis_users_user analysis_users ref qiita_user ( email ) email @@ -442,13 +447,13 @@ artifact ref software_command ( command_id ) command_id - + Fk fk_artifact_visibility artifact ref visibility ( visibility_id ) -Fk fk_artifact_visibility +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2176,784L 2176,864' ><title>Fk fk_artifact_visibility artifact ref visibility ( visibility_id ) -visibility_id +visibility_id Fk fk_artifact_filepath_artifact @@ -825,14 +830,6 @@ <path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 640,1392L 592,1392' ><title>Fk fk_study_tags per_study_tags ref study_tags ( study_tag ) study_tag - - - Fk fk_prep_template_artifact -prep_template ref artifact ( artifact_id ) - -Fk fk_prep_template_artifact -prep_template ref artifact ( artifact_id ) -artifact_id Fk fk_prep_template_data_type @@ -866,13 +863,13 @@ prep_template_processing_job ref prep_template ( prep_template_id ) prep_template_id - + Fk fk_prep_template_processing_job_job prep_template_processing_job ref processing_job ( processing_job_id ) -Fk fk_prep_template_processing_job_job +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2656,912L 2648,912Q 2640,912 2640,920L 2640,1032Q 2640,1040 2632,1040L 1880,1040Q 1872,1040 1872,1032L 1872,1000Q 1872,992 1864,992L 1800,992Q 1792,992 1792,984L 1792,984Q 1792,976 1784,976L 1776,976' ><title>Fk fk_prep_template_processing_job_job prep_template_processing_job ref processing_job ( processing_job_id ) -processing_job_id +processing_job_id Fk fk_prep_template @@ -898,13 +895,13 @@ preparation_artifact ref artifact ( artifact_id ) artifact_id - + Fk fk_prep_template_id preparation_artifact ref prep_template ( prep_template_id ) -Fk fk_prep_template_id +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2640,624L 2632,624Q 2624,624 2624,632L 2624,824Q 2624,832 2616,832L 2408,832Q 2400,832 2400,840L 2400,1272Q 2400,1280 2392,1280L 2272,1280' ><title>Fk fk_prep_template_id preparation_artifact ref prep_template ( prep_template_id ) -prep_template_id +prep_template_id Fk fk_processing_job_logging @@ -921,14 +918,6 @@ <path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 1520,832L 1456,832' ><title>Fk fk_processing_job_status processing_job ref processing_job_status ( processing_job_status_id ) processing_job_status_id - - - Fk fk_processing_job -processing_job ref software_command ( command_id ) - -Fk fk_processing_job -processing_job ref software_command ( command_id ) -command_id Fk fk_processing_job_validator_c @@ -962,13 +951,13 @@ processing_job_workflow_root ref processing_job_workflow ( processing_job_workflow_id ) processing_job_workflow_id - + Fk fk_user_user_level qiita_user ref user_level ( user_level_id ) -Fk fk_user_user_level +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 288,1504L 288,1528Q 288,1536 280,1536L 272,1536' ><title>Fk fk_user_user_level qiita_user ref user_level ( user_level_id ) -user_level_id +user_level_id Fk fk_reference_sequence_filepath @@ -994,29 +983,29 @@ reference ref filepath ( tree_filepath -> filepath_id ) tree_filepath - + Fk fk_filepath_id sample_template_filepath ref filepath ( filepath_id ) -Fk fk_filepath_id +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2896,1280L 2896,1160Q 2896,1152 2904,1152L 2928,1152' ><title>Fk fk_filepath_id sample_template_filepath ref filepath ( filepath_id ) filepath_id - + Fk fk_study_id sample_template_filepath ref study ( study_id ) -Fk fk_study_id +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 2704,1296L 2648,1296Q 2640,1296 2640,1288L 2640,1240Q 2640,1232 2632,1232L 1880,1232Q 1872,1232 1872,1240L 1872,1336Q 1872,1344 1864,1344L 360,1344Q 352,1344 352,1336L 352,872Q 352,864 344,864L 320,864' ><title>Fk fk_study_id sample_template_filepath ref study ( study_id ) study_id - + Fk fk_slurm_resource_allocations slurm_resource_allocations ref processing_job ( processing_job_id ) -Fk fk_slurm_resource_allocations +<path transform='translate(8,0)' marker-start='url(#foot1)' marker-end='url(#arrow1)' d='M 1872,784L 1776,784' ><title>Fk fk_slurm_resource_allocations slurm_resource_allocations ref processing_job ( processing_job_id ) -processing_job_id +processing_job_id Fk fk_software_software_type @@ -1090,19 +1079,19 @@ study ref timeseries_type ( timeseries_type_id ) timeseries_type_id - + Fk fk_study_artifact_artifact study_artifact ref artifact ( artifact_id ) -Fk fk_study_artifact_artifact +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 304,1104L 304,1096Q 304,1088 312,1088L 904,1088Q 912,1088 912,1096L 912,1304Q 912,1312 920,1312L 1928,1312Q 1936,1312 1936,1304L 1936,1080Q 1936,1072 1944,1072L 2104,1072Q 2112,1072 2112,1064L 2112,744Q 2112,736 2120,736L 2144,736' ><title>Fk fk_study_artifact_artifact study_artifact ref artifact ( artifact_id ) artifact_id - + Fk fk_study_artifact_study study_artifact ref study ( study_id ) -Fk fk_study_artifact_study +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 320,1136L 328,1136Q 336,1136 336,1128L 336,856Q 336,848 328,848L 320,848' ><title>Fk fk_study_artifact_study study_artifact ref study ( study_id ) study_id @@ -1146,27 +1135,27 @@ study_prep_template ref prep_template ( prep_template_id ) prep_template_id - + Fk fk_study_prep_template_study study_prep_template ref study ( study_id ) -Fk fk_study_prep_template_study +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 592,1104L 592,1080Q 592,1072 600,1072L 696,1072Q 704,1072 704,1064L 704,600Q 704,592 696,592L 632,592Q 624,592 624,584L 624,424Q 624,416 616,416L 312,416Q 304,416 304,424L 304,496' ><title>Fk fk_study_prep_template_study study_prep_template ref study ( study_id ) -study_id +study_id - + Fk fk_required_sample_info_study study_sample ref study ( study_id ) -Fk fk_required_sample_info_study +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 704,1200L 704,1112Q 704,1104 712,1104L 712,1104Q 720,1104 720,1096L 720,920Q 720,912 712,912L 392,912Q 384,912 384,920L 384,920Q 384,928 376,928L 280,928Q 272,928 272,920L 272,896' ><title>Fk fk_required_sample_info_study study_sample ref study ( study_id ) study_id - + Fk fk_email study_tags ref qiita_user ( email ) -Fk fk_email +<path transform='translate(8,0)' marker-start='url(#foot1p)' marker-end='url(#arrow1)' d='M 416,1376L 336,1376' ><title>Fk fk_email study_tags ref qiita_user ( email ) email @@ -1979,9 +1968,9 @@ * varchar varchar - + - + prep_templateTable qiita.prep_template Pk pk_prep_template ( prep_template_id ) prep_template_idprep_template_id * bigint default nextval('qiita.prep_template_prep_template_id_seq'::regclass) @@ -2015,7 +2004,9 @@ boolean default false boolean reprocess_job_idreprocess_job_id uuid default NULL -uuid +uuid human_reads_filter_method_idhuman_reads_filter_method_id +bigint default NULL +bigint @@ -2202,41 +2193,50 @@ varchar varchar - - - -qiita_userTable qiita.qiita_user +<rect class='entity' style='stroke:none;' x='48' y='1208' width='288' height='288' rx='8' ry='8' /> +<path d='M 48 1236 L 48 1216 Q 48 1208 56 1208 L 328 1208 Q 336 1208 336 1216 L 336 1236 L48 1236 ' style='fill:url(#tbg_D1BEF4);stroke:1;stroke-opacity:0.1;' /> +<rect class='entity' x='48' y='1208' width='288' height='288' rx='8' ry='8' style='fill:none;stroke:#28272C'/> +<a xlink:href='#qiita.qiita_user'><text x='157' y='1227' >qiita_user</text><title>Table qiita.qiita_user Holds all user information - Pk pk_user ( email ) emailemail + <use id='nn' x='50' y='1245' xlink:href='#nn'/><use id='pk' x='50' y='1244' xlink:href='#pk'><title>Pk pk_user ( email ) emailemail * varchar -varcharReferred by analysis_users ( email ) +<text x='320' y='1256' text-anchor='end' class='colType'>varchar</text><a xlink:href='#qiita.qiita_user.email'><use id='ref' x='324' y='1244' xlink:href='#ref'/><title>Referred by analysis_users ( email ) Referred by study_tags ( email ) - idx_user ( user_level_id ) user_level_iduser_level_id + <use id='nn' x='50' y='1261' xlink:href='#nn'/><use id='idx' x='50' y='1260' xlink:href='#idx'><title>idx_user ( user_level_id ) user_level_iduser_level_id * integer default 5 user level -integerReferences user_level ( user_level_id ) - passwordpassword +<text x='320' y='1272' text-anchor='end' class='colType'>integer</text><a xlink:href='#qiita.qiita_user.user_level_id'><use id='fk' x='324' y='1260' xlink:href='#fk'/><title>References user_level ( user_level_id ) + passwordpassword * varchar -varchar namename +<text x='320' y='1288' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_name'><text id='qiita.qiita_user.name' x='67' y='1303'>name</text><title>name varchar -varchar affiliationaffiliation +<text x='320' y='1304' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_affiliation'><text id='qiita.qiita_user.affiliation' x='67' y='1319'>affiliation</text><title>affiliation varchar -varchar addressaddress +<text x='320' y='1320' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_address'><text id='qiita.qiita_user.address' x='67' y='1335'>address</text><title>address varchar -varchar phonephone +<text x='320' y='1336' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_phone'><text id='qiita.qiita_user.phone' x='67' y='1351'>phone</text><title>phone varchar -varchar user_verify_codeuser_verify_code +<text x='320' y='1352' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_user_verify_code'><text id='qiita.qiita_user.user_verify_code' x='67' y='1367'>user_verify_code</text><title>user_verify_code varchar Code for initial user email verification -varchar pass_reset_codepass_reset_code +<text x='320' y='1368' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_pass_reset_code'><text id='qiita.qiita_user.pass_reset_code' x='67' y='1383'>pass_reset_code</text><title>pass_reset_code varchar Randomly generated code for password reset -varchar pass_reset_timestamppass_reset_timestamp +<text x='320' y='1384' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.qiita_user_pass_reset_timestamp'><text id='qiita.qiita_user.pass_reset_timestamp' x='67' y='1399'>pass_reset_timestamp</text><title>pass_reset_timestamp timestamp(0) Time the reset code was generated -timestamp(0) receive_processing_job_emailsreceive_processing_job_emails +<text x='320' y='1400' text-anchor='end' class='colType'>timestamp(0)</text> <use id='nn' x='50' y='1405' xlink:href='#nn'/><a xlink:href='#qiita.qiita_user_receive_processing_job_emails'><text id='qiita.qiita_user.receive_processing_job_emails' x='67' y='1415'>receive_processing_job_emails</text><title>receive_processing_job_emails * boolean default false -boolean +boolean social_orcidsocial_orcid +varchar default NULL +varchar social_researchgatesocial_researchgate +varchar default NULL +varchar social_googlescholarsocial_googlescholar +varchar default NULL +varchar creation_timestampcreation_timestamp +timestamp default NOW() +The date the user account was created +timestamp @@ -2322,26 +2322,32 @@ * varchar varchar - - - -slurm_resource_allocationsTable qiita.slurm_resource_allocations - Pk pk_slurm_resource_allocations_processing_job_id ( processing_job_id ) processing_job_idprocessing_job_id +<rect class='entity' style='stroke:none;' x='1888' y='616' width='192' height='208' rx='8' ry='8' /> +<path d='M 1888 644 L 1888 624 Q 1888 616 1896 616 L 2072 616 Q 2080 616 2080 624 L 2080 644 L1888 644 ' style='fill:url(#tbg_F4DDBE);stroke:1;stroke-opacity:0.1;' /> +<rect class='entity' x='1888' y='616' width='192' height='208' rx='8' ry='8' style='fill:none;stroke:#2C2927'/> +<a xlink:href='#qiita.slurm_resource_allocations'><text x='1894' y='635' >slurm_resource_allocations</text><title>Table qiita.slurm_resource_allocations + Pk pk_slurm_resource_allocations_processing_job_id ( processing_job_id ) processing_job_idprocessing_job_id * uuid -uuidReferences processing_job ( processing_job_id ) - samplessamples +<text x='2064' y='664' text-anchor='end' class='colType'>uuid</text><a xlink:href='#qiita.slurm_resource_allocations.processing_job_id'><use id='fk' x='2068' y='652' xlink:href='#fk'/><title>References processing_job ( processing_job_id ) + samplessamples integer -integer columnscolumns +<text x='2064' y='680' text-anchor='end' class='colType'>integer</text> <a xlink:href='#qiita.slurm_resource_allocations_columns'><text id='qiita.slurm_resource_allocations.columns' x='1907' y='695'>columns</text><title>columns integer -integer input_sizeinput_size +<text x='2064' y='696' text-anchor='end' class='colType'>integer</text> <a xlink:href='#qiita.slurm_resource_allocations_input_size'><text id='qiita.slurm_resource_allocations.input_size' x='1907' y='711'>input_size</text><title>input_size bigint -bigint extra_infoextra_info +<text x='2064' y='712' text-anchor='end' class='colType'>bigint</text> <a xlink:href='#qiita.slurm_resource_allocations_extra_info'><text id='qiita.slurm_resource_allocations.extra_info' x='1907' y='727'>extra_info</text><title>extra_info varchar default null -varchar memory_usedmemory_used +<text x='2064' y='728' text-anchor='end' class='colType'>varchar</text> <a xlink:href='#qiita.slurm_resource_allocations_memory_used'><text id='qiita.slurm_resource_allocations.memory_used' x='1907' y='743'>memory_used</text><title>memory_used bigint -bigint walltime_usedwalltime_used +<text x='2064' y='744' text-anchor='end' class='colType'>bigint</text> <a xlink:href='#qiita.slurm_resource_allocations_walltime_used'><text id='qiita.slurm_resource_allocations.walltime_used' x='1907' y='759'>walltime_used</text><title>walltime_used integer -integer +integer job_startjob_start +text +text node_namenode_name +varchar default NULL +varchar node_modelnode_model +varchar default NULL +varchar @@ -2715,6 +2721,16 @@ varchar visibility_descriptionvisibility_description * varchar varchar + + + + +human_reads_filter_methodTable qiita.human_reads_filter_method + Pk pk_human_reads_filter_method_human_reads_filter_method_id ( human_reads_filter_method_id ) human_reads_filter_method_idhuman_reads_filter_method_id +* bigint +bigint human_reads_filter_method_namehuman_reads_filter_method_name +* varchar +varchar
@@ -4331,6 +4347,30 @@
+

+
Table human_reads_filter_method
+ + + + + + + + + + + + + + + + + + + + +
IdxField NameData Type
*human_reads_filter_method_id bigint
*human_reads_filter_method_name varchar
Indexes
pk_human_reads_filter_method_human_reads_filter_method_id ON human_reads_filter_method_id
+

Table investigation

Overarching investigation information.An investigation comprises one or more individual studies.

@@ -4983,6 +5023,12 @@ uuid DEFAULT NULL + +   + human_reads_filter_method_id + bigint DEFAULT NULL + + Indexes pk_prep_template ON prep_template_id @@ -5514,7 +5560,7 @@


-
Table qiita_user
+
Table qiita_user

Holds all user information

@@ -5587,6 +5633,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -5821,7 +5891,7 @@
boolean DEFAULT false
 social_orcid varchar DEFAULT NULL
 social_researchgate varchar DEFAULT NULL
 social_googlescholar varchar DEFAULT NULL
 creation_timestamp timestamp DEFAULT NOW() The date the user account was created
Indexes
pk_user ON email


-
Table slurm_resource_allocations
+
Table slurm_resource_allocations
@@ -5862,6 +5932,21 @@ + + + + + + + + + + + + + + + From 0387611c89459a99e45b9e0327b7e739ab3649b6 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Tue, 2 Jul 2024 09:29:35 -0600 Subject: [PATCH 02/25] Human reads filter method (#3422) * Update CHANGELOG.md * human_reads_filter_method * address @charles-cowart comment and expose value to the GUI --- qiita_db/artifact.py | 48 + qiita_db/support_files/patches/92.sql | 15 +- .../support_files/patches/test_db_sql/92.sql | 10 + qiita_db/support_files/qiita-db.dbs | 191 +- qiita_db/support_files/qiita-db.html | 2760 +++++++++-------- qiita_db/test/test_artifact.py | 19 + .../handlers/study_handlers/prep_template.py | 7 + .../templates/study_ajax/prep_summary.html | 4 + 8 files changed, 1580 insertions(+), 1474 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 94f335c94..c19648276 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1684,3 +1684,51 @@ def get_commands(self): cids = cmds & cids return [qdb.software.Command(cid) for cid in cids] + + @property + def human_reads_filter_method(self): + """The human_reads_filter_method of the artifact + + Returns + ------- + str + The human_reads_filter_method name + """ + with qdb.sql_connection.TRN: + sql = """SELECT human_reads_filter_method + FROM qiita.artifact + LEFT JOIN qiita.human_reads_filter_method + USING (human_reads_filter_method_id) + WHERE artifact_id = %s""" + qdb.sql_connection.TRN.add(sql, [self.id]) + return qdb.sql_connection.TRN.execute_fetchlast() + + @human_reads_filter_method.setter + def human_reads_filter_method(self, value): + """Set the human_reads_filter_method of the artifact + + Parameters + ---------- + value : str + The new artifact's human_reads_filter_method + + Raises + ------ + ValueError + If `value` doesn't exist in the database + """ + with qdb.sql_connection.TRN: + sql = """SELECT human_reads_filter_method_id + FROM qiita.human_reads_filter_method + WHERE human_reads_filter_method = %s""" + qdb.sql_connection.TRN.add(sql, [value]) + idx = qdb.sql_connection.TRN.execute_fetchflatten() + + if len(idx) == 0: + raise ValueError( + f'"{value}" is not a valid human_reads_filter_method') + + sql = """UPDATE qiita.artifact + SET human_reads_filter_method_id = %s + WHERE artifact_id = %s""" + qdb.sql_connection.TRN.add(sql, [idx[0], self.id]) diff --git a/qiita_db/support_files/patches/92.sql b/qiita_db/support_files/patches/92.sql index b618c3133..37576bca2 100644 --- a/qiita_db/support_files/patches/92.sql +++ b/qiita_db/support_files/patches/92.sql @@ -48,19 +48,18 @@ ALTER TABLE qiita.qiita_user ADD social_orcid character varying DEFAULT NULL; ALTER TABLE qiita.qiita_user ADD social_researchgate character varying DEFAULT NULL; ALTER TABLE qiita.qiita_user ADD social_googlescholar character varying DEFAULT NULL; +-- Jul 1, 2024 -- Add human_reads_filter_method so we can keep track of the available methods -- and link them to the preparations -CREATE TABLE qiita.human_reads_filter_method ( - human_reads_filter_method_id bigint NOT NULL, - human_reads_filter_method_method character varying NOT NULL, - CONSTRAINT pk_human_reads_filter_method_id PRIMARY KEY ( - human_reads_filter_method_id ) - ); +CREATE TABLE qiita.human_reads_filter_method ( + human_reads_filter_method_id SERIAL PRIMARY KEY, + human_reads_filter_method character varying NOT NULL +); -ALTER TABLE qiita.prep_template +ALTER TABLE qiita.artifact ADD human_reads_filter_method_id bigint DEFAULT NULL; -ALTER TABLE qiita.prep_template +ALTER TABLE qiita.artifact ADD CONSTRAINT fk_human_reads_filter_method FOREIGN KEY ( human_reads_filter_method_id ) REFERENCES qiita.human_reads_filter_method ( human_reads_filter_method_id ); diff --git a/qiita_db/support_files/patches/test_db_sql/92.sql b/qiita_db/support_files/patches/test_db_sql/92.sql index 8346b18e1..938b1bb2a 100644 --- a/qiita_db/support_files/patches/test_db_sql/92.sql +++ b/qiita_db/support_files/patches/test_db_sql/92.sql @@ -942,3 +942,13 @@ WHERE email = 'test@foo.bar'; INSERT INTO qiita.qiita_user VALUES ('justnow@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'JustNow', 'NonVeriUser', '1634 Edgemont Avenue', '303-492-1984', NULL, NULL, NULL, false, NOW(), NULL, NULL, NULL); INSERT INTO qiita.qiita_user VALUES ('ayearago@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Oldie', 'NonVeriUser', '172 New Lane', '102-111-1984', NULL, NULL, NULL, false, NOW() - INTERVAL '1 YEAR', NULL, NULL, NULL); INSERT INTO qiita.qiita_user VALUES ('3Xdays@nonvalidat.ed', 5, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'TooLate', 'NonVeriUser', '564 C Street', '508-492-222', NULL, NULL, NULL, false, NOW() - INTERVAL '30 DAY', NULL, NULL, NULL); + +-- Jul 1, 2024 +-- Inserting a human_reads_filter_method and assigning it to the raw data in prep/artifact 1 +INSERT INTO qiita.human_reads_filter_method ( + human_reads_filter_method) + VALUES ( + 'The greatest human filtering method'); +UPDATE qiita.artifact + SET human_reads_filter_method_id = 1 + WHERE artifact_id = 1; diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index cf84177af..1a9d03043 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -221,6 +221,9 @@ + + + @@ -248,6 +251,9 @@ + + +
IdxField NameData Type
walltime_used integer
 job_start text
 node_name varchar DEFAULT NULL
 node_model varchar DEFAULT NULL
Indexes
pk_slurm_resource_allocations_processing_job_id ON processing_job_id
@@ -1014,9 +1020,6 @@ - - - @@ -2093,97 +2096,97 @@ $function$ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +{% end %} + +{% block content %} +
+ +
+

Please choose software, version, and command to view the data.

+
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+
+

Generated on: {{time}}

+
+ +
+ + + + + + + + + + + + +
Memory AllocationTime Allocation
+ {% if img_mem and img_mem is not None %} + + {% else %} + No memory allocation image available + {% end %} + + {% if img_time and img_time is not None %} + + {% else %} + No time allocation image available + {% end %} +
+
    +
  • k: {{ mk }}
  • +
  • a: {{ ma }}
  • +
  • b: {{ mb }}
  • +
  • model: {{ mmodel }}
  • +
  • real: {{ mreal }}
  • +
  • calc: {{ mcalc }}
  • +
  • fail: {{ mfail }}
  • +
+
+
    +
  • k: {{ tk }}
  • +
  • a: {{ ta }}
  • +
  • b: {{ tb }}
  • +
  • model: {{ tmodel }}
  • +
  • real: {{ treal }}
  • +
  • calc: {{ tcalc }}
  • +
  • fail: {{ tfail }}
  • +
+
+
+ +
+ + +{% end %} \ No newline at end of file diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 7165f33a2..5e2e0633f 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -384,6 +384,7 @@
  • View Studies awaiting approval
  • Edit study portal connections
  • Purge non-validated users
  • +
  • View Resource Allocation Plots
  • {% end %}
  • Sample Validation
  • Processing Jobs
  • diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 50b34fd2b..74367489b 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -50,6 +50,7 @@ from qiita_pet.handlers.upload import ( UploadFileHandler, StudyUploadFileHandler, StudyUploadViaRemote) from qiita_pet.handlers.stats import StatsHandler +from qiita_pet.handlers.resources import ResourcesHandler from qiita_pet.handlers.download import ( DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease, DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions, @@ -136,6 +137,7 @@ def __init__(self): (r"/admin/sample_validation/", SampleValidation), (r"/admin/purge_users/", PurgeUsersHandler), (r"/admin/purge_usersAjax/", PurgeUsersAJAXHandler), + (r"/admin/resources/", ResourcesHandler), (r"/ebi_submission/(.*)", EBISubmitHandler), # Study handlers (r"/study/create/", StudyEditHandler), From 49e4c1aa1aa736e9af219f497ba37df4457d5647 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 13 Sep 2024 11:16:26 -0600 Subject: [PATCH 12/25] doc improvemnts (#3434) --- CHANGELOG.md | 2 +- .../processingdata/processing-recommendations.rst | 2 +- .../doc/source/processingdata/woltka_pairedend.rst | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69df9a9a..03ad1593d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Deployed on September 23rd, 2024 * Initial changes in `qiita_client` to have more accurate variable names: `QIITA_SERVER_CERT` -> `QIITA_ROOTCA_CERT`. Thank you @charles-cowart! * Added `get_artifact_html_summary` to `qiita_client` to retrieve the summary file of an artifact. * Re-added github actions to `https://github.com/qiita-spots/qiita_client`. -* `Woltka v0.1.4, paired-end` superseded `Woltka v0.1.4` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! +* `Woltka v0.1.6, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425). diff --git a/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst b/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst index 14ce87f0f..0abc62e51 100755 --- a/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst +++ b/qiita_pet/support_files/doc/source/processingdata/processing-recommendations.rst @@ -125,7 +125,7 @@ Note that the command produces up to 5 output artifacts based on the aligner and .. note:: - Woltka 0.1.4 only produces per-genome, per-gene and functional profiles as we are moving + Woltka 0.1.6 only produces per-genome, per-gene and functional profiles as we are moving to Operational Genomic Units (OGUs), which have higher resolution than taxonomic units for community ecology, and were shown to deliver stronger biological signals in downstream analyses. For more information please read: `Phylogeny-Aware Analysis of diff --git a/qiita_pet/support_files/doc/source/processingdata/woltka_pairedend.rst b/qiita_pet/support_files/doc/source/processingdata/woltka_pairedend.rst index fe154b6d3..0084c61dc 100644 --- a/qiita_pet/support_files/doc/source/processingdata/woltka_pairedend.rst +++ b/qiita_pet/support_files/doc/source/processingdata/woltka_pairedend.rst @@ -6,16 +6,16 @@ Benchmarks created by Qiyun Zhu (@qiyunzhu) on Aug 1, 2024. Summary ------- -I tested alternative read pairing schemes in the analysis of shotgun metagenomic sequencing data. Sequencing reads were aligned against a reference microbial genome database as unpaired or paired, with or without singleton and/or discordant alignments suppressed. A series of synthetic datasets were used in the analysis. +I tested alternative read pairing schemes in the analysis of shotgun metagenomic sequencing data. Sequencing reads were aligned against a reference microbial genome database as unpaired or paired. A series of synthetic datasets were used in the analysis. -The results reveal that treating reads as paired is always advantageous over unpaired. Suppressing singleton alignments further increases the accuracy of results, despite the cost of lower mapping rate. Suppressing discordant alignments has no obvious impact on the result. Regardless of accuracy, the downstream community ecology analyses are not obviously impacted by the choice of parameters. +The results reveal that treating reads as paired is always advantageous over unpaired. Regardless of accuracy, the downstream community ecology analyses are not obviously impacted by the choice of parameters. -Therefore, I recommend the general adoption of paired alignments as a standard procedure. I also endorse suppressing singleton and discordant alignments, but note the favor of further tests on whether they may reduce sensitivity with complex communities. +Therefore, I recommend the general adoption of paired alignments as a standard procedure. Alignment parameters -------------------- -Sequencing data were aligned using Bowtie2 v2.5.1 in the “very sensitive” mode against the WoL2 database. They were treated as either unpaired or paired-end: +Sequencing data were aligned using Bowtie2 v2.5.1 in the "very sensitive" mode against the WoL2 database. They were treated as either unpaired or paired-end: - SE: Reads are treated as unpaired (Bowtie2 input: -U merged.fq) - PE: Reads are treated as paired (Bowtie2 input: -1 fwd.fq, -2 rev.fq) @@ -30,11 +30,10 @@ Five synthetic datasets were generated with 25 samples each consisting of random The results of the five Bowtie2 parameter sets were compared using nine metrics: -Three metrics that only rely on each result. +Two metrics that only rely on each result. - Mapping rate (%) - Number of taxa -- Entropy (i.e., Shannon index, but without subsampling) Six metrics that rely on comparing each result against the ground truth (higher is better): @@ -59,4 +58,4 @@ The results revealed: #. PE outperforms SE in all metrics. Most importantly, it reduces false positive rate (higher precision) while retaining mapping rate. Meanwhile, the sensitivity (recall) of identifying true taxa is not obviously compromised (note the y-axis scale). #. PE.NU the two additional parameters had minimum effect on the result and make the alignment step faster. This may suggest that the additional parameters are safe to use. -Therefore, I would recommend adopting paired alignment in preference to unpaired alignment. I may suggest no mixing as it has improved accuracy, but the potential adverse effect of lower mapping rate may be further explored before making a compelling recommendation. Although not having a visible effect, no discordance may be added for logical coherency. +Therefore, I would recommend adopting paired alignment in preference to unpaired alignment. From 25237a552df3be54b3ad42acc2ef7c3b932caa9e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 13:17:32 -0600 Subject: [PATCH 13/25] Doc improvements (#3437) * doc improvemnts * doc improvements * METAGENOMICS -> Metagenomics * 2024.10 * scp -O --- .github/workflows/qiita-ci.yml | 2 +- CHANGELOG.md | 8 +++++--- qiita_core/__init__.py | 2 +- qiita_db/__init__.py | 2 +- qiita_pet/__init__.py | 2 +- qiita_pet/handlers/api_proxy/__init__.py | 2 +- qiita_pet/support_files/doc/source/faq.rst | 7 ++++++- qiita_ware/__init__.py | 2 +- qiita_ware/ebi.py | 3 ++- qiita_ware/test/test_ebi.py | 10 +++++----- setup.py | 2 +- 11 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 51bae2a9b..dd596af87 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -179,7 +179,7 @@ jobs: echo "Connecting as $USER@localhost" # this line (and the -o StrictHostKeyChecking=no) is so the server # is added to the list of known servers - scp -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1 + scp -O -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1 - name: Main tests shell: bash -l {0} diff --git a/CHANGELOG.md b/CHANGELOG.md index 03ad1593d..41c4c821b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ # Qiita changelog -Version 2024.09 +Version 2024.10 --------------- -Deployed on September 23rd, 2024 +Deployed on October 14th, 2024 * Added update_resource_allocation_redis and companion code, so resource allocations summaries are available for review. Thank you @Gossty! * Now is possible to have default workflows with only one step. @@ -11,7 +11,9 @@ Deployed on September 23rd, 2024 * Initial changes in `qiita_client` to have more accurate variable names: `QIITA_SERVER_CERT` -> `QIITA_ROOTCA_CERT`. Thank you @charles-cowart! * Added `get_artifact_html_summary` to `qiita_client` to retrieve the summary file of an artifact. * Re-added github actions to `https://github.com/qiita-spots/qiita_client`. -* `Woltka v0.1.6, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! +* `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. +* `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. +* `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425). diff --git a/qiita_core/__init__.py b/qiita_core/__init__.py index e7cd25423..95d73a276 100644 --- a/qiita_core/__init__.py +++ b/qiita_core/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2024.09" +__version__ = "2024.10" diff --git a/qiita_db/__init__.py b/qiita_db/__init__.py index 165823462..814bbffaf 100644 --- a/qiita_db/__init__.py +++ b/qiita_db/__init__.py @@ -27,7 +27,7 @@ from . import user from . import processing_job -__version__ = "2024.09" +__version__ = "2024.10" __all__ = ["analysis", "artifact", "archive", "base", "commands", "environment_manager", "exceptions", "investigation", "logger", diff --git a/qiita_pet/__init__.py b/qiita_pet/__init__.py index e7cd25423..95d73a276 100644 --- a/qiita_pet/__init__.py +++ b/qiita_pet/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2024.09" +__version__ = "2024.10" diff --git a/qiita_pet/handlers/api_proxy/__init__.py b/qiita_pet/handlers/api_proxy/__init__.py index 0077522c0..b27c938f7 100644 --- a/qiita_pet/handlers/api_proxy/__init__.py +++ b/qiita_pet/handlers/api_proxy/__init__.py @@ -38,7 +38,7 @@ from .user import (user_jobs_get_req) from .util import check_access, check_fp -__version__ = "2024.09" +__version__ = "2024.10" __all__ = ['prep_template_summary_get_req', 'data_types_get_req', 'study_get_req', 'sample_template_filepaths_get_req', diff --git a/qiita_pet/support_files/doc/source/faq.rst b/qiita_pet/support_files/doc/source/faq.rst index 17fdf73f8..be56ae7b2 100755 --- a/qiita_pet/support_files/doc/source/faq.rst +++ b/qiita_pet/support_files/doc/source/faq.rst @@ -414,13 +414,18 @@ Are you planning a workshop or class? ------------------------------------------------------------------- We encourage users to use Qiita for their classes and/or workshops and to facilitate processing -we urge users to request a special reservation in the system. A reservation should help your +we urge them to request a special reservation in the system. A reservation should help your and your participant jobs to move quicker in the system. If you are interested, please send us an email to qiita.help@gmail.com and add the name of your workshop/course, the number of participants, the expected days this will happen. Note that reservations are only available for analysis and not for sequencing processing, and that the reservation can be added/edited during the creation of the analysis or at any point within each individual analysis page. +You can add the reservation string to the analysis at the time of creation (bottom of the creation +page), or the analysis page via the reservation button (top right). Note that once the reservation +time finishes, your jobs will fail as the reservation will no longer exist so you will need to +remove it - we suggest doing this before submitting new jobs. + How to cite Qiita? ------------------ diff --git a/qiita_ware/__init__.py b/qiita_ware/__init__.py index e7cd25423..95d73a276 100644 --- a/qiita_ware/__init__.py +++ b/qiita_ware/__init__.py @@ -6,4 +6,4 @@ # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -__version__ = "2024.09" +__version__ = "2024.10" diff --git a/qiita_ware/ebi.py b/qiita_ware/ebi.py index ec6b084c9..7c7861ec4 100644 --- a/qiita_ware/ebi.py +++ b/qiita_ware/ebi.py @@ -110,6 +110,7 @@ class EBISubmission(object): 'ILLUMINA MISEQ', 'ILLUMINA MINISEQ', 'ILLUMINA NOVASEQ 6000', + 'ILLUMINA NOVASEQ X', 'NEXTSEQ 500', 'NEXTSEQ 550', 'UNSPECIFIED'], @@ -539,7 +540,7 @@ def generate_experiment_xml(self, samples=None): library_name.text = self._get_library_name(sample_name) lg = ET.SubElement(library_descriptor, 'LIBRARY_STRATEGY') - lg.text = escape(clean_whitespace(library_strategy.upper())) + lg.text = escape(clean_whitespace(library_strategy)) # hardcoding some values, # see https://github.com/biocore/qiita/issues/1485 diff --git a/qiita_ware/test/test_ebi.py b/qiita_ware/test/test_ebi.py index 7ef1b0b08..e28fd39d3 100644 --- a/qiita_ware/test/test_ebi.py +++ b/qiita_ware/test/test_ebi.py @@ -1358,7 +1358,7 @@ def test_parse_EBI_reply(self): %(study_id)s.Sample1" /> %(study_id)s.Sample1 - METAGENOMICS + Metagenomics METAGENOMIC PCR @@ -1393,7 +1393,7 @@ def test_parse_EBI_reply(self): %(study_id)s.Sample2" /> %(study_id)s.Sample2 - METAGENOMICS + Metagenomics METAGENOMIC PCR @@ -1428,7 +1428,7 @@ def test_parse_EBI_reply(self): %(study_id)s.Sample3" /> %(study_id)s.Sample3 - METAGENOMICS + Metagenomics METAGENOMIC PCR @@ -1469,7 +1469,7 @@ def test_parse_EBI_reply(self): 1.SKB2.640194 - METAGENOMICS + Metagenomics METAGENOMIC PCR @@ -1558,7 +1558,7 @@ def test_parse_EBI_reply(self): 1.SKB3.640195 - METAGENOMICS + Metagenomics METAGENOMIC PCR diff --git a/setup.py b/setup.py index 1b7d111d6..230cec3a2 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from setuptools import setup from glob import glob -__version__ = "2024.09" +__version__ = "2024.10" classes = """ From 8c3e3428e49abcea3ca0c5bd9b250404fb2c9c10 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 14:18:02 -0600 Subject: [PATCH 14/25] fix #3438 (#3439) --- CHANGELOG.md | 2 +- qiita_pet/templates/study_ajax/prep_summary.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41c4c821b..438075906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Deployed on October 14th, 2024 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! -* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425). +* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439). Version 2024.07 diff --git a/qiita_pet/templates/study_ajax/prep_summary.html b/qiita_pet/templates/study_ajax/prep_summary.html index c1b823291..25674e8a1 100644 --- a/qiita_pet/templates/study_ajax/prep_summary.html +++ b/qiita_pet/templates/study_ajax/prep_summary.html @@ -434,7 +434,7 @@

    {{name}} - ID {{prep_id}} ({{data_type}}) {% if user_level in ('admin', 'wet-lab admin') and creation_job is not None %} - SampleSheet + SampleSheet {% if creation_job_artifact_summary is not None %} Creation Job Output {% end %} From 35a051e10b7c71ab482a21dfc191d2ffdf6fe1c8 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 11 Oct 2024 10:22:43 -0600 Subject: [PATCH 15/25] Fix 3436 (#3440) * fix #3438 * fix #3436 * self->cls * rm () * adding tests * fix test * fix flake8 --- CHANGELOG.md | 4 +- qiita_db/artifact.py | 3 +- qiita_db/metadata_template/prep_template.py | 12 +++++ qiita_db/test/test_artifact.py | 49 +++++++++++++++++++-- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 438075906..d29e6491d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ Deployed on October 14th, 2024 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! -* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439). +* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). +* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). + Version 2024.07 diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index f116236f7..bf81ddf41 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self): # words has more that one processing step behind it fine_to_send = [] fine_to_send.extend([pt.artifact for pt in self.prep_templates]) - fine_to_send.extend([c for a in fine_to_send for c in a.children]) + fine_to_send.extend([c for a in fine_to_send if a is not None + for c in a.children]) if self not in fine_to_send: return False diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 7ae97858c..52f781b9a 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -272,6 +272,18 @@ def delete(cls, id_): "Cannot remove prep template %d because it has an artifact" " associated with it" % id_) + # artifacts that are archived are not returned as part of the code + # above and we need to clean them before moving forward + sql = """SELECT artifact_id + FROM qiita.preparation_artifact + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, args) + archived_artifacts = set( + qdb.sql_connection.TRN.execute_fetchflatten()) + if archived_artifacts: + for aid in archived_artifacts: + qdb.artifact.Artifact.delete(aid) + # Delete the prep template filepaths sql = """DELETE FROM qiita.prep_template_filepath WHERE prep_template_id = %s""" diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 76833ed98..bed71c45a 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -10,7 +10,7 @@ from tempfile import mkstemp, mkdtemp from datetime import datetime from os import close, remove -from os.path import exists, join, basename +from os.path import exists, join, basename, dirname, abspath from shutil import copyfile from functools import partial from json import dumps @@ -23,6 +23,7 @@ from qiita_core.util import qiita_test_checker from qiita_core.testing import wait_for_processing_job import qiita_db as qdb +from qiita_ware.private_plugin import _delete_analysis_artifacts class ArtifactTestsReadOnly(TestCase): @@ -1518,7 +1519,7 @@ def test_archive(self): 'be archived'): A.archive(8) - for aid in range(4, 7): + for aid in range(5, 7): ms = A(aid).merging_scheme A.archive(aid) self.assertEqual(ms, A(aid).merging_scheme) @@ -1526,7 +1527,49 @@ def test_archive(self): self.assertCountEqual(A(1).descendants.nodes(), exp_nodes) obs_artifacts = len(qdb.util.get_artifacts_information([4, 5, 6, 8])) - self.assertEqual(1, obs_artifacts) + self.assertEqual(2, obs_artifacts) + + # in the tests above we generated and validated archived artifacts + # so this allows us to add tests to delete a prep-info with archived + # artifacts. The first bottleneck to do this is that this tests will + # actually remove files, which we will need for other tests so lets + # make a copy and then restore them + mfolder = dirname(dirname(abspath(__file__))) + mpath = join(mfolder, 'support_files', 'test_data') + mp = partial(join, mpath) + fps = [ + mp('processed_data/1_study_1001_closed_reference_otu_table.biom'), + mp('processed_data/' + '1_study_1001_closed_reference_otu_table_Silva.biom'), + mp('raw_data/1_s_G1_L001_sequences.fastq.gz'), + mp('raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz')] + for fp in fps: + copyfile(fp, f'{fp}.bk') + + PT = qdb.metadata_template.prep_template.PrepTemplate + QEE = qdb.exceptions.QiitaDBExecutionError + pt = A(1).prep_templates[0] + # it should fail as this prep is public and have been submitted to ENA + with self.assertRaisesRegex(QEE, 'Cannot remove prep template 1'): + PT.delete(pt.id) + # now, remove those restrictions + analysis + linked artifacts + sql = "DELETE FROM qiita.artifact_processing_job" + qdb.sql_connection.perform_as_transaction(sql) + sql = "DELETE FROM qiita.ebi_run_accession" + qdb.sql_connection.perform_as_transaction(sql) + sql = "UPDATE qiita.artifact SET visibility_id = 1" + qdb.sql_connection.perform_as_transaction(sql) + _delete_analysis_artifacts(qdb.analysis.Analysis(1)) + _delete_analysis_artifacts(qdb.analysis.Analysis(2)) + _delete_analysis_artifacts(qdb.analysis.Analysis(3)) + for aid in [3, 2, 1]: + A.delete(aid) + + PT.delete(pt.id) + + # bringing back the filepaths + for fp in fps: + copyfile(f'{fp}.bk', fp) if __name__ == '__main__': From c0e715ba16569a768e04508f944aef458daf59d3 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sat, 12 Oct 2024 07:56:48 -0600 Subject: [PATCH 16/25] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d29e6491d..33fb0303a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Deployed on October 14th, 2024 * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). * General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). +* We updated the available parameters for `Filter features against reference [filter_features]`, `Non V4 16S sequence assessment [non_v4_16s]` and all the phylogenetic analytical commands so they can use `Greengenes2 2024.09`. From edd3eed7474f0c78da594a152b2d16cc14f848fa Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 18 Oct 2024 13:00:59 -0600 Subject: [PATCH 17/25] Improve complete job (#3443) * Update CHANGELOG.md * improve complete_job * CREATE pg_trgm * EXTENSION IF NOT EXISTS * fix tests * SELECT change * = -> ILIKE --- CHANGELOG.md | 1 + qiita_db/handlers/processing_job.py | 4 +- qiita_db/processing_job.py | 77 ++++++++++++++------------- qiita_db/support_files/patches/93.sql | 57 ++++++++++++++++++++ 4 files changed, 100 insertions(+), 39 deletions(-) create mode 100644 qiita_db/support_files/patches/93.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index d29e6491d..33fb0303a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Deployed on October 14th, 2024 * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). * General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). +* We updated the available parameters for `Filter features against reference [filter_features]`, `Non V4 16S sequence assessment [non_v4_16s]` and all the phylogenetic analytical commands so they can use `Greengenes2 2024.09`. diff --git a/qiita_db/handlers/processing_job.py b/qiita_db/handlers/processing_job.py index 6bb15cdf4..832d2407a 100644 --- a/qiita_db/handlers/processing_job.py +++ b/qiita_db/handlers/processing_job.py @@ -146,7 +146,9 @@ def post(self, job_id): cmd, values_dict={'job_id': job_id, 'payload': self.request.body.decode( 'ascii')}) - job = qdb.processing_job.ProcessingJob.create(job.user, params) + # complete_job are unique so it is fine to force them to be created + job = qdb.processing_job.ProcessingJob.create( + job.user, params, force=True) job.submit() self.finish() diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 11145925b..a8844d181 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -582,10 +582,10 @@ def create(cls, user, parameters, force=False): TTRN = qdb.sql_connection.TRN with TTRN: command = parameters.command - - # check if a job with the same parameters already exists - sql = """SELECT processing_job_id, email, processing_job_status, - COUNT(aopj.artifact_id) + if not force: + # check if a job with the same parameters already exists + sql = """SELECT processing_job_id, email, + processing_job_status, COUNT(aopj.artifact_id) FROM qiita.processing_job LEFT JOIN qiita.processing_job_status USING (processing_job_status_id) @@ -596,41 +596,42 @@ def create(cls, user, parameters, force=False): GROUP BY processing_job_id, email, processing_job_status""" - # we need to use ILIKE because of booleans as they can be - # false or False - params = [] - for k, v in parameters.values.items(): - # this is necessary in case we have an Iterable as a value - # but that is string - if isinstance(v, Iterable) and not isinstance(v, str): - for vv in v: - params.extend([k, str(vv)]) + # we need to use ILIKE because of booleans as they can be + # false or False + params = [] + for k, v in parameters.values.items(): + # this is necessary in case we have an Iterable as a value + # but that is string + if isinstance(v, Iterable) and not isinstance(v, str): + for vv in v: + params.extend([k, str(vv)]) + else: + params.extend([k, str(v)]) + + if params: + # divided by 2 as we have key-value pairs + len_params = int(len(params)/2) + sql = sql.format(' AND ' + ' AND '.join( + ["command_parameters->>%s ILIKE %s"] * len_params)) + params = [command.id] + params + TTRN.add(sql, params) else: - params.extend([k, str(v)]) - - if params: - # divided by 2 as we have key-value pairs - len_params = int(len(params)/2) - sql = sql.format(' AND ' + ' AND '.join( - ["command_parameters->>%s ILIKE %s"] * len_params)) - params = [command.id] + params - TTRN.add(sql, params) - else: - # the sql variable expects the list of parameters but if there - # is no param we need to replace the {0} with an empty string - TTRN.add(sql.format(""), [command.id]) - - # checking that if the job status is success, it has children - # [2] status, [3] children count - existing_jobs = [r for r in TTRN.execute_fetchindex() - if r[2] != 'success' or r[3] > 0] - if existing_jobs and not force: - raise ValueError( - 'Cannot create job because the parameters are the same as ' - 'jobs that are queued, running or already have ' - 'succeeded:\n%s' % '\n'.join( - ["%s: %s" % (jid, status) - for jid, _, status, _ in existing_jobs])) + # the sql variable expects the list of parameters but if + # there is no param we need to replace the {0} with an + # empty string + TTRN.add(sql.format(""), [command.id]) + + # checking that if the job status is success, it has children + # [2] status, [3] children count + existing_jobs = [r for r in TTRN.execute_fetchindex() + if r[2] != 'success' or r[3] > 0] + if existing_jobs: + raise ValueError( + 'Cannot create job because the parameters are the ' + 'same as jobs that are queued, running or already ' + 'have succeeded:\n%s' % '\n'.join( + ["%s: %s" % (jid, status) + for jid, _, status, _ in existing_jobs])) sql = """INSERT INTO qiita.processing_job (email, command_id, command_parameters, diff --git a/qiita_db/support_files/patches/93.sql b/qiita_db/support_files/patches/93.sql new file mode 100644 index 000000000..81abc3331 --- /dev/null +++ b/qiita_db/support_files/patches/93.sql @@ -0,0 +1,57 @@ +-- Oct 18, 2024 +-- ProcessingJob.create can take up to 52 seconds if creating a complete_job; mainly +-- due to the number of jobs of this command and using json. The solution in the database +-- is to convert to jsonb and index the values of the database + +-- ### This are the stats before the change in a single example +-- GroupAggregate (cost=67081.81..67081.83 rows=1 width=77) (actual time=51859.962..51862.637 rows=1 loops=1) +-- Group Key: processing_job.processing_job_id, processing_job_status.processing_job_status +-- -> Sort (cost=67081.81..67081.81 rows=1 width=77) (actual time=51859.952..51862.627 rows=1 loops=1) +-- Sort Key: processing_job.processing_job_id, processing_job_status.processing_job_status +-- Sort Method: quicksort Memory: 25kB +-- -> Nested Loop Left Join (cost=4241.74..67081.80 rows=1 width=77) (actual time=51859.926..51862.604 rows=1 loops=1) +-- -> Nested Loop (cost=4237.30..67069.64 rows=1 width=69) (actual time=51859.889..51862.566 rows=1 loops=1) +-- Join Filter: (processing_job.processing_job_status_id = processing_job_status.processing_job_status_id) +-- Rows Removed by Join Filter: 1 +-- -> Gather (cost=4237.30..67068.50 rows=1 width=45) (actual time=51859.846..51862.522 rows=1 loops=1) +-- Workers Planned: 2 +-- Workers Launched: 2 +-- -> Parallel Bitmap Heap Scan on processing_job (cost=3237.30..66068.40 rows=1 width=45) (actual time=51785.317..51785.446 rows=0 loops=3) +-- Recheck Cond: (command_id = 83) +-- Filter: (((command_parameters ->> 'job_id'::text) ~~* '3432a908-f7b8-4e36-89fc-88f3310b84d5'::text) AND ((command_parameters ->> ' +-- payload'::text) ~~* '{"success": true, "error": "", "artifacts": {"alpha_diversity": {"artifact_type": "alpha_vector", "filepaths": [["/qmounts/qiita_test_data/tes +-- tlocal/working_dir/3432a908-f7b8-4e36-89fc-88f3310b84d5/alpha_phylogenetic/alpha_diversity/alpha-diversity.tsv", "plain_text"], ["/qmounts/qiita_test_data/testloca +-- l/working_dir/3432a908-f7b8-4e36-89fc-88f3310b84d5/alpha_phylogenetic/alpha_diversity.qza", "qza"]], "archive": {}}}}'::text)) +-- Rows Removed by Filter: 97315 +-- Heap Blocks: exact=20133 +-- -> Bitmap Index Scan on idx_processing_job_command_id (cost=0.00..3237.30 rows=294517 width=0) (actual time=41.569..41.569 rows= +-- 293054 loops=1) +-- Index Cond: (command_id = 83) +-- -> Seq Scan on processing_job_status (cost=0.00..1.09 rows=4 width=40) (actual time=0.035..0.035 rows=2 loops=1) +-- Filter: ((processing_job_status)::text = ANY ('{success,waiting,running,in_construction}'::text[])) +-- Rows Removed by Filter: 1 +-- -> Bitmap Heap Scan on artifact_output_processing_job aopj (cost=4.43..12.14 rows=2 width=24) (actual time=0.031..0.031 rows=0 loops=1) +-- Recheck Cond: (processing_job.processing_job_id = processing_job_id) +-- -> Bitmap Index Scan on idx_artifact_output_processing_job_job (cost=0.00..4.43 rows=2 width=0) (actual time=0.026..0.026 rows=0 loops=1) +-- Index Cond: (processing_job_id = processing_job.processing_job_id) +-- Planning Time: 1.173 ms +-- Execution Time: 51862.756 ms + +-- Note: for this to work you need to have created as admin the extension +-- CREATE EXTENSION pg_trgm; +CREATE EXTENSION IF NOT EXISTS "pg_trgm" WITH SCHEMA public; + +-- This alter table will take close to 11 min +ALTER TABLE qiita.processing_job + ALTER COLUMN command_parameters TYPE JSONB USING command_parameters::jsonb; + +-- This indexing will take like 5 min +CREATE INDEX IF NOT EXISTS processing_job_command_parameters_job_id ON qiita.processing_job + USING GIN((command_parameters->>'job_id') gin_trgm_ops); + +-- This indexing will take like an hour +CREATE INDEX IF NOT EXISTS processing_job_command_parameters_payload ON qiita.processing_job + USING GIN((command_parameters->>'payload') gin_trgm_ops); + +-- After the changes +-- 18710.404 ms From ea0a7ecf793634412235e06e9532227ec3673076 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 11 Dec 2024 13:50:33 -0700 Subject: [PATCH 18/25] some improvements (#3445) * some changes * rm circular import * AMPLICON * AMPLICON tests * reservation * fix errors * fix qiita_db tests * fix trace * addressing @charles-cowart comments --- qiita_db/analysis.py | 16 +++++++++++++ .../handlers/tests/test_processing_job.py | 4 ++-- qiita_db/metadata_template/prep_template.py | 16 ++++++++++++- .../test/test_prep_template.py | 6 ++--- qiita_db/processing_job.py | 12 ++++++---- qiita_db/support_files/patches/93.sql | 7 ++++++ qiita_db/support_files/populate_test_db.sql | 2 +- qiita_db/test/test_analysis.py | 2 +- qiita_db/test/test_artifact.py | 7 +++--- qiita_db/test/test_ontology.py | 2 +- qiita_db/util.py | 14 +++++++---- .../tests/test_base_handlers.py | 2 +- .../api_proxy/tests/test_prep_template.py | 6 ++--- qiita_pet/templates/resources.html | 24 ++++++++----------- qiita_ware/private_plugin.py | 14 ++--------- scripts/qiita-recover-jobs | 3 ++- 16 files changed, 84 insertions(+), 53 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index c7e44855f..dc9126691 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -215,6 +215,22 @@ def create(cls, owner, name, description, from_default=False, job.submit() return instance + @classmethod + def delete_analysis_artifacts(cls, _id): + """Deletes the artifacts linked to an artifact and then the analysis + + Parameters + ---------- + _id : int + The analysis id + """ + analysis = cls(_id) + aids = [a.id for a in analysis.artifacts if not a.parents] + aids.sort(reverse=True) + for aid in aids: + qdb.artifact.Artifact.delete(aid) + cls.delete(analysis.id) + @classmethod def delete(cls, _id): """Deletes an analysis diff --git a/qiita_db/handlers/tests/test_processing_job.py b/qiita_db/handlers/tests/test_processing_job.py index 5ef82669a..b747b1f3e 100644 --- a/qiita_db/handlers/tests/test_processing_job.py +++ b/qiita_db/handlers/tests/test_processing_job.py @@ -233,9 +233,9 @@ def test_post_job_success(self): self.assertIsNotNone(cj) # additionally we can test that job.print_trace is correct self.assertEqual(job.trace, [ - f'{job.id} [Not Available]: Validate | ' + f'{job.id} [Not Available] (success): Validate | ' '-p qiita -N 1 -n 1 --mem 90gb --time 150:00:00 --nice=10000', - f' {cj.id} [{cj.external_id}] | ' + f' {cj.id} [{cj.external_id}] (success)| ' '-p qiita -N 1 -n 1 --mem 16gb --time 10:00:00 --nice=10000']) def test_post_job_success_with_archive(self): diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 52f781b9a..117156d9b 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -135,7 +135,7 @@ def create(cls, md_template, study, data_type, investigation_type=None, # data_type being created - if possible if investigation_type is None: if data_type_str in TARGET_GENE_DATA_TYPES: - investigation_type = 'Amplicon' + investigation_type = 'AMPLICON' elif data_type_str == 'Metagenomic': investigation_type = 'WGS' elif data_type_str == 'Metatranscriptomic': @@ -280,8 +280,22 @@ def delete(cls, id_): qdb.sql_connection.TRN.add(sql, args) archived_artifacts = set( qdb.sql_connection.TRN.execute_fetchflatten()) + ANALYSIS = qdb.analysis.Analysis if archived_artifacts: for aid in archived_artifacts: + # before we can delete the archived artifact, we need + # to delete the analyses where they were used. + sql = """SELECT analysis_id + FROM qiita.analysis + WHERE analysis_id IN ( + SELECT DISTINCT analysis_id + FROM qiita.analysis_sample + WHERE artifact_id IN %s)""" + qdb.sql_connection.TRN.add(sql, [tuple([aid])]) + analyses = set( + qdb.sql_connection.TRN.execute_fetchflatten()) + for _id in analyses: + ANALYSIS.delete_analysis_artifacts(_id) qdb.artifact.Artifact.delete(aid) # Delete the prep template filepaths diff --git a/qiita_db/metadata_template/test/test_prep_template.py b/qiita_db/metadata_template/test/test_prep_template.py index ea41694fe..c4978f47b 100644 --- a/qiita_db/metadata_template/test/test_prep_template.py +++ b/qiita_db/metadata_template/test/test_prep_template.py @@ -911,7 +911,7 @@ def _common_creation_checks(self, pt, fp_count, name): self.assertEqual(pt.data_type(), self.data_type) self.assertEqual(pt.data_type(ret_id=True), self.data_type_id) self.assertEqual(pt.artifact, None) - self.assertEqual(pt.investigation_type, 'Amplicon') + self.assertEqual(pt.investigation_type, 'AMPLICON') self.assertEqual(pt.study_id, self.test_study.id) self.assertEqual(pt.status, "sandbox") exp_sample_ids = {'%s.SKB8.640193' % self.test_study.id, @@ -1076,7 +1076,7 @@ def test_create_warning(self): self.assertEqual(pt.data_type(), self.data_type) self.assertEqual(pt.data_type(ret_id=True), self.data_type_id) self.assertEqual(pt.artifact, None) - self.assertEqual(pt.investigation_type, 'Amplicon') + self.assertEqual(pt.investigation_type, 'AMPLICON') self.assertEqual(pt.study_id, self.test_study.id) self.assertEqual(pt.status, 'sandbox') exp_sample_ids = {'%s.SKB8.640193' % self.test_study.id, @@ -1247,7 +1247,7 @@ def test_investigation_type_setter(self): """Able to update the investigation type""" pt = qdb.metadata_template.prep_template.PrepTemplate.create( self.metadata, self.test_study, self.data_type_id) - self.assertEqual(pt.investigation_type, 'Amplicon') + self.assertEqual(pt.investigation_type, 'AMPLICON') pt.investigation_type = "Other" self.assertEqual(pt.investigation_type, 'Other') with self.assertRaises(qdb.exceptions.QiitaDBColumnError): diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index a8844d181..27192bab7 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -2053,23 +2053,25 @@ def complete_processing_job(self): def trace(self): """ Returns as a text array the full trace of the job, from itself to validators and complete jobs""" - lines = [f'{self.id} [{self.external_id}]: ' + lines = [f'{self.id} [{self.external_id}] ({self.status}): ' f'{self.command.name} | {self.resource_allocation_info}'] cjob = self.complete_processing_job if cjob is not None: - lines.append(f' {cjob.id} [{cjob.external_id}] | ' + lines.append(f' {cjob.id} [{cjob.external_id}] ({cjob.status})| ' f'{cjob.resource_allocation_info}') vjob = self.release_validator_job if vjob is not None: lines.append(f' {vjob.id} [{vjob.external_id}] ' - f'| {vjob.resource_allocation_info}') + f' ({vjob.status}) | ' + f'{vjob.resource_allocation_info}') for v in self.validator_jobs: - lines.append(f' {v.id} [{v.external_id}]: ' + lines.append(f' {v.id} [{v.external_id}] ({v.status}): ' f'{v.command.name} | {v.resource_allocation_info}') cjob = v.complete_processing_job if cjob is not None: lines.append(f' {cjob.id} [{cjob.external_id}] ' - f'| {cjob.resource_allocation_info}') + f'({cjob.status}) | ' + f'{cjob.resource_allocation_info}') return lines diff --git a/qiita_db/support_files/patches/93.sql b/qiita_db/support_files/patches/93.sql index 81abc3331..4befc74d6 100644 --- a/qiita_db/support_files/patches/93.sql +++ b/qiita_db/support_files/patches/93.sql @@ -55,3 +55,10 @@ CREATE INDEX IF NOT EXISTS processing_job_command_parameters_payload ON qiita.pr -- After the changes -- 18710.404 ms + +-- + +-- Nov 5, 2024 +-- Addding contraints for the slurm_reservation column +ALTER TABLE qiita.analysis DROP CONSTRAINT IF EXISTS analysis_slurm_reservation_valid_chars; +ALTER TABLE qiita.analysis ADD CONSTRAINT analysis_slurm_reservation_valid_chars CHECK ( slurm_reservation ~ '^[a-zA-Z0-9_]*$' ); diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index ea7c4337c..a1f0095d0 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -1336,7 +1336,7 @@ INSERT INTO qiita.study_users VALUES (1, 'shared@foo.bar'); INSERT INTO qiita.term VALUES (2052508974, 999999999, NULL, 'WGS', 'ENA:0000059', NULL, NULL, NULL, NULL, NULL, false); INSERT INTO qiita.term VALUES (2052508975, 999999999, NULL, 'Metagenomics', 'ENA:0000060', NULL, NULL, NULL, NULL, NULL, false); -INSERT INTO qiita.term VALUES (2052508976, 999999999, NULL, 'Amplicon', 'ENA:0000061', NULL, NULL, NULL, NULL, NULL, false); +INSERT INTO qiita.term VALUES (2052508976, 999999999, NULL, 'AMPLICON', 'ENA:0000061', NULL, NULL, NULL, NULL, NULL, false); INSERT INTO qiita.term VALUES (2052508984, 999999999, NULL, 'RNA-Seq', 'ENA:0000070', NULL, NULL, NULL, NULL, NULL, false); INSERT INTO qiita.term VALUES (2052508987, 999999999, NULL, 'Other', 'ENA:0000069', NULL, NULL, NULL, NULL, NULL, false); diff --git a/qiita_db/test/test_analysis.py b/qiita_db/test/test_analysis.py index a43811997..d428ca8fd 100644 --- a/qiita_db/test/test_analysis.py +++ b/qiita_db/test/test_analysis.py @@ -688,7 +688,7 @@ def test_is_public_make_public(self): def test_slurm_reservation(self): analysis = qdb.analysis.Analysis(1) self.assertIsNone(analysis.slurm_reservation) - text = 'this is a test!' + text = 'thisisatest' analysis.slurm_reservation = text self.assertEqual(analysis._slurm_reservation(), [text]) self.assertIsNone(analysis.slurm_reservation) diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index bed71c45a..2319acfaa 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -23,7 +23,6 @@ from qiita_core.util import qiita_test_checker from qiita_core.testing import wait_for_processing_job import qiita_db as qdb -from qiita_ware.private_plugin import _delete_analysis_artifacts class ArtifactTestsReadOnly(TestCase): @@ -1559,9 +1558,9 @@ def test_archive(self): qdb.sql_connection.perform_as_transaction(sql) sql = "UPDATE qiita.artifact SET visibility_id = 1" qdb.sql_connection.perform_as_transaction(sql) - _delete_analysis_artifacts(qdb.analysis.Analysis(1)) - _delete_analysis_artifacts(qdb.analysis.Analysis(2)) - _delete_analysis_artifacts(qdb.analysis.Analysis(3)) + qdb.analysis.Analysis.delete_analysis_artifacts(1) + qdb.analysis.Analysis.delete_analysis_artifacts(2) + qdb.analysis.Analysis.delete_analysis_artifacts(3) for aid in [3, 2, 1]: A.delete(aid) diff --git a/qiita_db/test/test_ontology.py b/qiita_db/test/test_ontology.py index c7b1fca25..483c0dc06 100644 --- a/qiita_db/test/test_ontology.py +++ b/qiita_db/test/test_ontology.py @@ -35,7 +35,7 @@ def testShortNameProperty(self): def testTerms(self): obs = self.ontology.terms self.assertEqual( - obs, ['WGS', 'Metagenomics', 'Amplicon', 'RNA-Seq', 'Other']) + obs, ['WGS', 'Metagenomics', 'AMPLICON', 'RNA-Seq', 'Other']) def test_user_defined_terms(self): obs = self.ontology.user_defined_terms diff --git a/qiita_db/util.py b/qiita_db/util.py index 0dae03431..c7346e15a 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2334,7 +2334,7 @@ def send_email(to, subject, body): msg = MIMEMultipart() msg['From'] = qiita_config.smtp_email msg['To'] = to - msg['Subject'] = subject + msg['Subject'] = subject.strip() msg.attach(MIMEText(body, 'plain')) # connect to smtp server, using ssl if needed @@ -2496,9 +2496,9 @@ def _resource_allocation_plot_helper( ax.set_ylabel(curr) ax.set_xlabel(col_name) - # 100 - number of maximum iterations, 3 - number of failures we tolerate + # 50 - number of maximum iterations, 3 - number of failures we tolerate best_model, options = _resource_allocation_calculate( - df, x_data, y_data, models, curr, col_name, 100, 3) + df, x_data, y_data, models, curr, col_name, 50, 3) k, a, b = options.x x_plot = np.array(sorted(df[col_name].unique())) y_plot = best_model(x_plot, k, a, b) @@ -2593,6 +2593,8 @@ def _resource_allocation_calculate( failures_df = _resource_allocation_failures( df, k, a, b, model, col_name, type_) y_plot = model(x, k, a, b) + if not any(y_plot): + continue cmax = max(y_plot) cmin = min(y_plot) failures = failures_df.shape[0] @@ -2834,13 +2836,17 @@ def merge_rows(rows): wait_time = ( datetime.strptime(rows.iloc[0]['Start'], date_fmt) - datetime.strptime(rows.iloc[0]['Submit'], date_fmt)) - tmp = rows.iloc[1].copy() + if rows.shape[0] >= 2: + tmp = rows.iloc[1].copy() + else: + tmp = rows.iloc[0].copy() tmp['WaitTime'] = wait_time return tmp slurm_data['external_id'] = slurm_data['JobID'].apply( lambda x: int(x.split('.')[0])) slurm_data['external_id'] = slurm_data['external_id'].ffill() + slurm_data = slurm_data.groupby( 'external_id').apply(merge_rows).reset_index(drop=True) diff --git a/qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py b/qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py index dec54fb9c..65d48e12c 100644 --- a/qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py +++ b/qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py @@ -207,7 +207,7 @@ def test_patch(self): self.assertEqual(analysis._slurm_reservation(), ['']) # now, let's change it to something different - reservation = 'my-reservation' + reservation = 'myreservation' arguments = { 'op': 'replace', 'path': 'reservation', 'value': reservation} self.patch(f'/analysis/description/{analysis.id}/', data=arguments) diff --git a/qiita_pet/handlers/api_proxy/tests/test_prep_template.py b/qiita_pet/handlers/api_proxy/tests/test_prep_template.py index a9f96ca9a..51a5d33b1 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_prep_template.py +++ b/qiita_pet/handlers/api_proxy/tests/test_prep_template.py @@ -38,7 +38,7 @@ class TestPrepAPIReadOnly(TestCase): def test_get_ENA_ontology(self): obs = _get_ENA_ontology() exp = { - 'ENA': ['Amplicon', 'Metagenomics', 'RNA-Seq', 'WGS', 'Other'], + 'ENA': ['AMPLICON', 'Metagenomics', 'RNA-Seq', 'WGS', 'Other'], 'User': []} self.assertEqual(obs, exp) @@ -53,7 +53,7 @@ def test_new_prep_template_get_req(self): 'Multiomic', 'Proteomic', 'Transcriptomics', 'Viromics'], 'ontology': { - 'ENA': ['Amplicon', 'Metagenomics', 'RNA-Seq', 'WGS', 'Other'], + 'ENA': ['AMPLICON', 'Metagenomics', 'RNA-Seq', 'WGS', 'Other'], 'User': []}} self.assertEqual(obs, exp) @@ -73,7 +73,7 @@ def test_prep_template_ajax_get_req(self): 'num_columns': 22, 'investigation_type': 'Metagenomics', 'ontology': { - 'ENA': ['Amplicon', 'Metagenomics', 'RNA-Seq', 'WGS', + 'ENA': ['AMPLICON', 'Metagenomics', 'RNA-Seq', 'WGS', 'Other'], 'User': []}, 'artifact_attached': True, diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html index d4b9e870c..e044429d2 100644 --- a/qiita_pet/templates/resources.html +++ b/qiita_pet/templates/resources.html @@ -21,14 +21,14 @@

    Please choose software, version, and command to view the data.

    - +
    - +
    -

    - {{analysis_name}} - ID {{analysis_id}} - - ({{analysis_description}}) - - {% if analysis_is_public %} - Public - {% else %} - Private - {% end %} - {% if analysis_mapping_id is not None %} - Mapping file - {% end %} - {% if not analysis_is_public %} - Make analysis public - {% end %} - Reservation: {% raw analysis_reservation %} - - - -

    - Shared with: -
    -
    - Studies and artifacts used in this analysis: - -
    - - - - - - - - - - - - - - {% for aid, data in artifacts.items() %} - - - - - - - - - - {% end %} - -
    Artifact IDStudy IDsPrep IDsStudy TitleParent ProcessingMerging SchemeTotal Samples Selected
    {{aid}}{{data[0]}}{{', '.join(data[4])}}{{data[1]}}{{data[2][1]}}{{data[2][0]}}{{ len(data[3]) }}
    +
    +
    + +
    +

    + {{analysis_name}} - ID {{analysis_id}} + + ({{analysis_description}}) - {% if analysis_is_public %} + Public + {% else %} + Private + {% end %} {% if analysis_mapping_id is not None %} + + Mapping file + {% end %} {% if not analysis_is_public %} + Make + analysis public + {% end %} + + Reservation: {% raw analysis_reservation %} + +

    + Owner: + {{analysis_owner}} +
    + + Shared with: +
    +
    + Studies and artifacts used in this analysis: + +
    + + + + + + + + + + + + + + {% for aid, data in artifacts.items() %} + + + + + + + + + + {% end %} + +
    Artifact IDStudy IDsPrep IDsStudy TitleParent ProcessingMerging SchemeTotal Samples Selected
    {{aid}}{{data[0]}}{{', '.join(data[4])}} + {{data[1]}} + {{data[2][1]}}{{data[2][0]}}{{ len(data[3]) }}
    +
    +
    -
    -
    -
    +
    -
    - +
    +
    -
    - +
    - -