diff --git a/DemoData/prep_data.txt b/DemoData/prep_data.txt index 6ea0328..54f5cc3 100644 --- a/DemoData/prep_data.txt +++ b/DemoData/prep_data.txt @@ -1,4 +1,4 @@ -sample_name barcode center_name center_project_name experiment_design_description instrument_model library_construction_protocol linker pcr_primers platform primer run_center run_date run_prefix sequencing_meth target_gene target_subfragment -6.15299.zr5156.1V3V4 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_1V3V4 Sequencing by synthesis 16S rRNA V3 -6.15299.zr5156.2V3V4 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_2V3V4 Sequencing by synthesis 16S rRNA V3 -6.15299.zr5156.3V3V4 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_3V3V4 Sequencing by synthesis 16S rRNA V3 +sample_name plate_id well_id barcode center_name center_project_name experiment_design_description instrument_model library_construction_protocol linker pcr_primers platform primer run_center run_date run_prefix sequencing_meth target_gene target_subfragment +6.15299.zr5156.1V3V4 1 A1 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_1V3V4 Sequencing by synthesis 16S rRNA V3 +6.15299.zr5156.2V3V4 1 B2 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_2V3V4 Sequencing by synthesis 16S rRNA V3 +6.15299.zr5156.3V3V4 1 C1 not provided Zymo Research Corporation zr5156.16S_221128.zymo FLASH vs conventional irradiation on gut microbiome Illumina MiSeq DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA). GT FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC Illumina CCTAYGGGDBGCWGCAG GSI Helmhotzzentrum fuer Schwerionenforschung GmbH 2022-11-28 zr5156_3V3V4 Sequencing by synthesis 16S rRNA V3 diff --git a/Images/qp-platemapper/qp-platemapper.dockerfile b/Images/qp-platemapper/qp-platemapper.dockerfile new file mode 100644 index 0000000..5f8b16b --- /dev/null +++ b/Images/qp-platemapper/qp-platemapper.dockerfile @@ -0,0 +1,77 @@ +FROM ubuntu:24.04 + +ARG MINIFORGE_VERSION=24.1.2-0 + +ENV CONDA_DIR=/opt/conda +ENV PATH=${CONDA_DIR}/bin:${PATH} + +RUN apt-get -y update +RUN apt-get -y --fix-missing install \ + git \ + wget \ + libpq-dev \ + python3-dev \ + gcc \ + build-essential \ + zip + +# install miniforge3 for "conda" +# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile +RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \ + /bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc && \ + conda init && \ + rm -f /tmp/miniforge3.sh + +# install tornado based trigger layer in base environment +RUN pip install -U pip +RUN conda install tornado +COPY trigger.py /trigger.py + +# Download qiime2 yaml +RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml + +# Create conda env +RUN conda env create --name platemapper -y --file qiime2-2023.5-py38-linux-conda.yml +# Make RUN commands use the new environment: +# append --format docker to the build command, see https://github.com/containers/podman/issues/8477 +SHELL ["conda", "run", "-p", "/opt/conda/envs/platemapper", "/bin/bash", "-c"] + +RUN pip install -U pip +RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip +RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip +RUN pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip +RUN git clone https://github.com/qiita-spots/qp-qiime2.git +WORKDIR qp-qiime2 +RUN pip install -e . +RUN pip install --upgrade certifi +RUN pip install pip-system-certs + +WORKDIR / +RUN git clone -b make_qiita_plugin https://github.com/jlab/qp-platemapper.git +WORKDIR qp-platemapper +RUN rm -f pyproject.toml +RUN pip install -e . + +WORKDIR / + +COPY start_qp-platemapper.sh . +RUN chmod 755 start_qp-platemapper.sh + +RUN mkdir -p /unshared_plugins +ENV QIITA_PLUGINS_DIR=/unshared_plugins/ + +## Export cert and config filepaths +COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem +#RUN cat /unshared_certificates/stefan_rootca.crt >> `python -c "import certifi; print(certifi.where())"` # append own rootCA onto chain of trust +ENV REQUESTS_CA_BUNDLE=/qiita_server_certificates/qiita_server_certificates.pem +ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem + +#RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt +RUN chmod u+x /qp-platemapper/scripts/configure_platemapper /qp-platemapper/scripts/start_platemapper +COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ +RUN /qp-platemapper/scripts/configure_platemapper --env-script 'true' --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-platemapper/" /unshared_plugins/*.conf + +CMD ["./start_qp-platemapper.sh"] diff --git a/Images/qp-platemapper/start_qp-platemapper.sh b/Images/qp-platemapper/start_qp-platemapper.sh new file mode 100644 index 0000000..4921f0c --- /dev/null +++ b/Images/qp-platemapper/start_qp-platemapper.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd / && python trigger.py platemapper start_platemapper /qp-platemapper + +tail -f /dev/null diff --git a/Makefile b/Makefile index c50dfc6..3db1af5 100644 --- a/Makefile +++ b/Makefile @@ -97,13 +97,20 @@ plugin: Images/qtp-biom/trigger.py $(DIR_REFERENCES)/qiita_server_certificates cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita touch .built_image_qiita +.built_image_qp-platemapper: Images/qp-platemapper/qp-platemapper.dockerfile Images/qp-platemapper/start_qp-platemapper.sh + test -d src/qp-platemapper || git clone https://github.com/jlab/qp-platemapper.git src/qp-platemapper + tmpdir=$(TMPDIR) $(MAKE) plugin + cp $^ $(TMPDIR) + $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` + touch .built_image_`basename $< | cut -d "." -f 1` + .built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh tmpdir=$(TMPDIR) $(MAKE) plugin cp $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-plugin_collector touch .built_image_plugin_collector -images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder +images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder .built_image_qp-platemapper environments/qiita_db.env: environments/qiita_db.env.example cp environments/qiita_db.env.example environments/qiita_db.env diff --git a/compose.yaml b/compose.yaml index 1af06e5..8bd0808 100644 --- a/compose.yaml +++ b/compose.yaml @@ -297,6 +297,23 @@ services: networks: - qiita-net + qp-platemapper: + image: local-qp-platemapper:latest + command: ['./start_qp-platemapper.sh'] + # network_mode: host + # stdin_open: true + # tty: true + restart: no + volumes: + - qiita-data:/qiita_data + - ./src/qp-platemapper:/qp-platemapper:U + - ./references/qiita_server_certificates:/qiita_server_certificates + environment: + - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - TZ=Europe/Berlin # this is important to avoid a local timezone error! See https://forum.qiime2.org/t/qiime2-timezone-error/17410 + networks: + - qiita-net + qtp-job-output-folder: image: local-qtp-job-output-folder:latest command: ['./start_qtp-job-output-folder.sh'] @@ -342,8 +359,10 @@ services: condition: service_started qtp-job-output-folder: condition: service_started + qp-platemapper: + condition: service_started environment: - - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:" + - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:qp-platemapper:" command: ['/startup_plugin_collector.sh'] networks: