Adding qp platemapper

DemoData/prep_data.txt

@@ -1,4 +1,4 @@
-sample_name	barcode	center_name	center_project_name	experiment_design_description	instrument_model	library_construction_protocol	linker	pcr_primers	platform	primer	run_center	run_date	run_prefix	sequencing_meth	target_gene	target_subfragment
-6.15299.zr5156.1V3V4	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_1V3V4	Sequencing by synthesis	16S rRNA	V3
-6.15299.zr5156.2V3V4	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_2V3V4	Sequencing by synthesis	16S rRNA	V3
-6.15299.zr5156.3V3V4	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_3V3V4	Sequencing by synthesis	16S rRNA	V3
+sample_name	plate_id	well_id	barcode	center_name	center_project_name	experiment_design_description	instrument_model	library_construction_protocol	linker	pcr_primers	platform	primer	run_center	run_date	run_prefix	sequencing_meth	target_gene	target_subfragment
+6.15299.zr5156.1V3V4	1	A1	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_1V3V4	Sequencing by synthesis	16S rRNA	V3
+6.15299.zr5156.2V3V4	1	B2	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_2V3V4	Sequencing by synthesis	16S rRNA	V3
+6.15299.zr5156.3V3V4	1	C1	not provided	Zymo Research Corporation	zr5156.16S_221128.zymo	FLASH vs conventional irradiation on gut microbiome	Illumina MiSeq	DNA Extraction: One of three different DNA extraction kits was used depending on the sample type and sample volume. In most cases, the ZymoBIOMICS®-96 MagBead DNA Kit (Zymo Research, Irvine, CA) was used to extract DNA using an automated platform. In some cases, ZymoBIOMICS® DNA Miniprep Kit (Zymo Research, Irvine, CA) was used. For low biomass samples, such as skin swabs, the ZymoBIOMICS® DNA Microprep Kit (Zymo Research, Irvine, CA) was used as it permits for a lower elution volume, resulting in more concentrated DNA samples. Targeted Library Preparation: Bacterial 16S ribosomal RNA gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit (Zymo Research, Irvine, CA). In most cases, the bacterial 16S primers amplified the V3-V4 region of the 16S rRNA gene. These primers have been custom-designed by Zymo Research to provide the best coverage of the 16S gene while maintaining high sensitivity. Fungal ITS gene targeted sequencing was performed using the Quick-16S(TM) NGS Library Prep Kit with custom ITS2 primers substituted for 16S primers. The sequencing library was prepared using an innovative library preparation process in which PCR reactions were performed in real-time PCR machines to control cycles and therefore limit PCR chimera formation. The final PCR products were quantified with qPCR fluorescence readings and pooled together based on equal molarity. The final pooled library was cleaned with the Select-a-Size DNA Clean & Concentrator(TM) (Zymo Research, Irvine, CA), then quantified with TapeStation®(Agilent Technologies, Santa Clara, CA) and Qubit® (Thermo Fisher Scientific, Waltham, WA).	GT	FWD:CCTAYGGGDBGCWGCAG; REV:GACTACNVGGGTMTCTAATCC	Illumina	CCTAYGGGDBGCWGCAG	GSI Helmhotzzentrum fuer Schwerionenforschung GmbH	2022-11-28	zr5156_3V3V4	Sequencing by synthesis	16S rRNA	V3

Images/qp-platemapper/qp-platemapper.dockerfile

@@ -0,0 +1,77 @@
+FROM ubuntu:24.04
+
+ARG MINIFORGE_VERSION=24.1.2-0
+
+ENV CONDA_DIR=/opt/conda
+ENV PATH=${CONDA_DIR}/bin:${PATH}
+
+RUN apt-get -y  update
+RUN apt-get -y --fix-missing install \
+	git \
+	wget \
+	libpq-dev \
+	python3-dev \
+	gcc \
+	build-essential \
+	zip
+
+# install miniforge3 for "conda"
+# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \
+	/bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \
+	echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
+	echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc && \
+	conda init && \
+	rm -f /tmp/miniforge3.sh
+
+# install tornado based trigger layer in base environment
+RUN pip install -U pip
+RUN conda install tornado
+COPY trigger.py /trigger.py
+
+# Download qiime2 yaml
+RUN wget --quiet https://data.qiime2.org/distro/core/qiime2-2023.5-py38-linux-conda.yml
+
+# Create conda env
+RUN conda env create --name platemapper -y --file qiime2-2023.5-py38-linux-conda.yml
+# Make RUN commands use the new environment:
+# append --format docker to the build command, see https://github.com/containers/podman/issues/8477
+SHELL ["conda", "run", "-p", "/opt/conda/envs/platemapper", "/bin/bash", "-c"]
+
+RUN pip install -U pip
+RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
+RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
+RUN pip install https://github.com/biocore/q2-mislabeled/archive/refs/heads/main.zip
+RUN git clone https://github.com/qiita-spots/qp-qiime2.git
+WORKDIR qp-qiime2
+RUN pip install -e .
+RUN pip install --upgrade certifi
+RUN pip install pip-system-certs
+
+WORKDIR /
+RUN git clone -b    make_qiita_plugin https://github.com/jlab/qp-platemapper.git
+WORKDIR qp-platemapper
+RUN rm -f pyproject.toml
+RUN pip install -e .
+
+WORKDIR /
+
+COPY start_qp-platemapper.sh .
+RUN chmod 755 start_qp-platemapper.sh
+
+RUN mkdir -p /unshared_plugins
+ENV QIITA_PLUGINS_DIR=/unshared_plugins/
+
+##  Export cert and config filepaths
+COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem
+#RUN cat /unshared_certificates/stefan_rootca.crt >> `python -c "import certifi; print(certifi.where())"`  # append own rootCA onto chain of trust
+ENV REQUESTS_CA_BUNDLE=/qiita_server_certificates/qiita_server_certificates.pem
+ENV SSL_CERT_FILE=/qiita_server_certificates/qiita_server_certificates.pem
+
+#RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt
+RUN chmod u+x /qp-platemapper/scripts/configure_platemapper /qp-platemapper/scripts/start_platemapper
+COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
+RUN /qp-platemapper/scripts/configure_platemapper  --env-script 'true' --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
+RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-platemapper/" /unshared_plugins/*.conf
+
+CMD ["./start_qp-platemapper.sh"]

Images/qp-platemapper/start_qp-platemapper.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd / && python trigger.py platemapper start_platemapper /qp-platemapper
+
+tail -f /dev/null

Makefile

@@ -97,13 +97,20 @@ plugin: Images/qtp-biom/trigger.py $(DIR_REFERENCES)/qiita_server_certificates
 	cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita
 	touch .built_image_qiita
 
+.built_image_qp-platemapper: Images/qp-platemapper/qp-platemapper.dockerfile Images/qp-platemapper/start_qp-platemapper.sh
+	test -d src/qp-platemapper || git clone https://github.com/jlab/qp-platemapper.git src/qp-platemapper
+	tmpdir=$(TMPDIR) $(MAKE) plugin
+	cp $^ $(TMPDIR)
+	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
+	touch .built_image_`basename $< | cut -d "." -f 1`
+
 .built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh
 	tmpdir=$(TMPDIR) $(MAKE) plugin
 	cp $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-plugin_collector
 	touch .built_image_plugin_collector
 
-images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder
+images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder .built_image_qp-platemapper
 
 environments/qiita_db.env: environments/qiita_db.env.example
 	cp environments/qiita_db.env.example environments/qiita_db.env

compose.yaml

@@ -297,6 +297,23 @@ services:
     networks:
       - qiita-net
 
+  qp-platemapper:
+    image: local-qp-platemapper:latest
+    command: ['./start_qp-platemapper.sh']
+    # network_mode: host
+    # stdin_open: true
+    # tty: true
+    restart: no
+    volumes:
+      - qiita-data:/qiita_data
+      - ./src/qp-platemapper:/qp-platemapper:U
+      - ./references/qiita_server_certificates:/qiita_server_certificates
+    environment:
+      - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
+      - TZ=Europe/Berlin  # this is important to avoid a local timezone error! See https://forum.qiime2.org/t/qiime2-timezone-error/17410
+    networks:
+      - qiita-net
+
   qtp-job-output-folder:
     image: local-qtp-job-output-folder:latest
     command: ['./start_qtp-job-output-folder.sh']
@@ -342,8 +359,10 @@ services:
         condition: service_started
       qtp-job-output-folder:
         condition: service_started
+      qp-platemapper:
+        condition: service_started
     environment:
-      - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:"
+      - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:qp-platemapper:"
     command: ['/startup_plugin_collector.sh']
 
 networks: