diff --git a/Images/qp-multiqc/qp-multiqc.dockerfile b/Images/qp-multiqc/qp-multiqc.dockerfile new file mode 100644 index 0000000..7383bee --- /dev/null +++ b/Images/qp-multiqc/qp-multiqc.dockerfile @@ -0,0 +1,72 @@ +FROM ubuntu:24.04 + +ARG MINIFORGE_VERSION=24.1.2-0 + +ENV CONDA_DIR=/opt/conda +ENV PATH=${CONDA_DIR}/bin:${PATH} + +RUN apt-get -y update +RUN apt-get -y --fix-missing install \ + git \ + wget \ + libpq-dev \ + python3-dev \ + gcc \ + build-essential + +# install miniforge3 for "conda" +# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile +RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \ + /bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc && \ + conda init && \ + rm -f /tmp/miniforge3.sh + +# install tornado based trigger layer in base environment +RUN pip install -U pip +RUN conda install tornado +COPY trigger.py /trigger.py + +# Create conda env +RUN conda create --quiet -n multiqc -c bioconda python=3.9 pip fastqc multiqc +# Make RUN commands use the new environment: +# append --format docker to the build command, see https://github.com/containers/podman/issues/8477 +SHELL ["conda", "run", "-p", "/opt/conda/envs/multiqc", "/bin/bash", "-c"] + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +RUN pip install -U pip + +RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip +RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip +RUN git clone https://github.com/jlab/qp-multiqc.git +WORKDIR qp-multiqc +RUN pip install -e . +RUN pip install pip-system-certs + +# TODO: should the plugin get the server configuration?! +RUN export QIITA_CONFIG_FP=/qiita/config_qiita_oidc.cfg + +WORKDIR / + +COPY start_qp-multiqc.sh . +RUN chmod 755 start_qp-multiqc.sh + +RUN mkdir -p /unshared_plugins +ENV QIITA_PLUGINS_DIR=/unshared_plugins/ + +## Export cert and config filepaths +COPY Certificates /unshared_certificates +RUN cat /unshared_certificates/stefan_rootca.crt >> `python -c "import certifi; print(certifi.where())"` # append own rootCA onto chain of trust +RUN export REQUESTS_CA_BUNDLE=`python -c "import certifi; print(certifi.where())"` +RUN export SSL_CERT_FILE=`python -c "import certifi; print(certifi.where())"` + +#RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt +#RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/" $CONDA_PREFIX/lib/python3.5/site-packages/qiita_client/plugin.py +RUN chmod u+x /qp-multiqc/scripts/configure_qp_multiqc +RUN /qp-multiqc/scripts/configure_qp_multiqc --env-script "true" --server-cert /unshared_certificates/stefan_server.crt +RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-multiqc/" /unshared_plugins/*.conf + +CMD ["./start_qp-multiqc.sh"] diff --git a/Images/qp-multiqc/start_qp-multiqc.sh b/Images/qp-multiqc/start_qp-multiqc.sh new file mode 100644 index 0000000..ede745d --- /dev/null +++ b/Images/qp-multiqc/start_qp-multiqc.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd / && python trigger.py multiqc start_qp_multiqc /qp-multiqc + +tail -f /dev/null diff --git a/Makefile b/Makefile index 90927b7..c7de8b6 100644 --- a/Makefile +++ b/Makefile @@ -31,52 +31,52 @@ Certificates/: Images/plugin_collector/stefan_csr.conf Images/plugin_collector/s plugin: Images/qtp-biom/trigger.py Certificates/ cp -r $^ $(tmpdir)/ -.built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile Images/qtp-biom/start_qtp-biom.sh +.built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile Images/qtp-biom/start_qtp-biom.sh Certificates/* test -d src/qtp-biom || git clone https://github.com/qiita-spots/qtp-biom.git src/qtp-biom tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qtp-sequencing: Images/qtp-sequencing/qtp-sequencing.dockerfile Images/qtp-sequencing/start_qtp-sequencing.sh +.built_image_qtp-sequencing: Images/qtp-sequencing/qtp-sequencing.dockerfile Images/qtp-sequencing/start_qtp-sequencing.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh +.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qtp-visualization: Images/qtp-visualization/qtp-visualization.dockerfile Images/qtp-visualization/start_qtp-visualization.sh +.built_image_qtp-visualization: Images/qtp-visualization/qtp-visualization.dockerfile Images/qtp-visualization/start_qtp-visualization.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qtp-diversity: Images/qtp-diversity/qtp-diversity.dockerfile Images/qtp-diversity/start_qtp-diversity.sh +.built_image_qtp-diversity: Images/qtp-diversity/qtp-diversity.dockerfile Images/qtp-diversity/start_qtp-diversity.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qp-deblur: Images/qp-deblur/qp-deblur.dockerfile Images/qp-deblur/start_qp-deblur.sh +.built_image_qp-deblur: Images/qp-deblur/qp-deblur.dockerfile Images/qp-deblur/start_qp-deblur.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qp-qiime2: Images/qp-qiime2/qp-qiime2.dockerfile Images/qp-qiime2/start_qp-qiime2.sh +.built_image_qp-qiime2: Images/qp-qiime2/qp-qiime2.dockerfile Images/qp-qiime2/start_qp-qiime2.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` -.built_image_qtp-job-output-folder: Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile Images/qtp-job-output-folder/start_qtp-job-output-folder.sh +.built_image_qtp-job-output-folder: Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile Images/qtp-job-output-folder/start_qtp-job-output-folder.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin - cp $^ $(TMPDIR) + cp -r $^ $(TMPDIR) $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` touch .built_image_`basename $< | cut -d "." -f 1` @@ -94,14 +94,20 @@ plugin: Images/qtp-biom/trigger.py Certificates/ cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita touch .built_image_qiita -.built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh +.built_image_qp-multiqc: Images/qp-multiqc/qp-multiqc.dockerfile Images/qp-multiqc/start_qp-multiqc.sh Certificates/* + tmpdir=$(TMPDIR) $(MAKE) plugin + cp -r $^ $(TMPDIR) + $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1` + touch .built_image_`basename $< | cut -d "." -f 1` + +.built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh Certificates/* tmpdir=$(TMPDIR) $(MAKE) plugin cp $^ $(TMPDIR) cp -r Certificates/ $(tmpdir)/ $(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-plugin_collector touch .built_image_plugin_collector -images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder +images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder .built_image_qp-multiqc environments/qiita_db.env: environments/qiita_db.env.example cp environments/qiita_db.env.example environments/qiita_db.env diff --git a/compose.yaml b/compose.yaml index dd53596..a9b7d50 100644 --- a/compose.yaml +++ b/compose.yaml @@ -343,6 +343,25 @@ services: networks: - qiita-net + qp-multiqc: + image: local-qp-multiqc:latest + command: ['./start_qp-multiqc.sh'] + # network_mode: host + # stdin_open: true + # tty: true + restart: no + volumes: + - qiita-data:/qiita_data + - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r # TODO: do we really want to expose server settings to the plugin? + environment: + # TODO: is there a more elegant way to obtain this path? + - REQUESTS_CA_BUNDLE=/opt/conda/envs/multiqc/lib/python3.9/site-packages/certifi/cacert.pem + - SSL_CERT_FILE=/opt/conda/envs/multiqc/lib/python3.9/site-packages/certifi/cacert.pem + - QIITA_CLIENT_DEBUG_LEVEL=DEBUG + - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg + networks: + - qiita-net + plugin-collector: # prior to qiita (master and worker) start up, iterates through the QIITA_PLUGINS : separated list of plugin containers # to compile all q*.conf files from plugin containers in the server-plugin-configs volume @@ -374,8 +393,10 @@ services: condition: service_started qtp-job-output-folder: condition: service_started + qp-multiqc: + condition: service_started environment: - - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:" + - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:qp-multiqc:" command: ['/startup_plugin_collector.sh'] networks: