Uncouple qtp sequencing

.github/workflows/buildContainer.yaml

@@ -162,7 +162,9 @@ jobs:
       - name: Execute tests in the running services
         run: |
           sleep 5
-          docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
+          fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "jlab"; else echo "qiita-spots"; fi`
+          branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "uncouplePlugin"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi`
+          docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh"
 
       - name: Push image to ghcr (only if tests passed)
         if: success()

Images/qiita/qiita.dockerfile

@@ -47,9 +47,14 @@ RUN pip install \
 	psycopg2-binary
 
 
-# Clone the Qiita Repo
+# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing)
 # RUN git clone -b master https://github.com/qiita-spots/qiita.git
-RUN git clone -b auth_oidc https://github.com/jlab/qiita.git
+RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \
+	&& cd qiita \
+	&& git config pull.rebase false \
+	&& git config --global user.email "[email protected]" \
+	&& git config --global user.name "Stefan" && \
+	git pull origin tornado_FetchFileFromCentralHandler
 
 # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env
 RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql

Images/qp-deblur/qp-deblur.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.22
+# VERSION: 2025.09.12
 
 # ==========================
 # Stage 1: Build wheels
@@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"]
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git
 RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/"  qiita_client/qiita_client/plugin.py
 RUN cd qiita_client && pip install --no-cache-dir .
 
@@ -130,7 +131,7 @@ RUN cd qiita_client && pip install .
 
 RUN mkdir -p /qiita_server_certificates/
 COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
-RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem
+RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https
 RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf
 
 # remove conda command from tigger.py

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.08.28
+# VERSION: 2025.09.17
 
 # ==========================
 # Stage 1: Build wheels
@@ -44,11 +44,13 @@ RUN python2.7 get-pip2.7.py --force-reinstall
 
 RUN pip install -U pip
 #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git
+
 RUN cd /qiita_client && pip install --no-cache-dir .
 
 RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
-RUN git clone https://github.com/qiita-spots/qp-target-gene.git
+RUN git clone -b uncouplePlugin https://github.com/jlab/qp-target-gene.git
 WORKDIR /qp-target-gene
 RUN pip install biom-format
 RUN pip install -e .
@@ -127,7 +129,7 @@ RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt
 COPY qiita_server_certificates/*_server.* /qiita_server_certificates/
 RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/configure_target_gene
 RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/start_target_gene
-RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f`
+RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https
 RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-target-gene/" /unshared_plugins/*.conf
 
 # for testing

Images/qtp-sequencing/qtp-sequencing.dockerfile

@@ -1,4 +1,4 @@
-# VERSION: 2025.09.08
+# VERSION: 2025.09.29
 
 # ==========================
 # Stage 1: Build wheels (~5.8 GB)
@@ -36,14 +36,17 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-sequencing", "/bin/bash", "-c"
 
 RUN pip install -U pip
 #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
-RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git
+RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git
+
 RUN cd qiita_client && pip install --no-cache-dir .
 
 # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
 RUN git clone -b master https://github.com/qiita-spots/qiita-files.git
 RUN cd /qiita-files && pip install -e . -v
 
-RUN git clone https://github.com/qiita-spots/qtp-sequencing.git
+#RUN git clone https://github.com/qiita-spots/qtp-sequencing.git
+RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-sequencing.git
 WORKDIR /qtp-sequencing
 RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py
 RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py

Images/test_plugin.sh

@@ -2,6 +2,11 @@
 
 echo "plugin to be tested is: '$PLUGIN'"
 
+# by default, clone from master. But can also clone different branches
+branch="${PLUGIN_BRANCH:-master}"
+# by default, clone from qiita-spots branch. But can also use alternative forks
+fork="${PLUGIN_FORK:-qiita-spots}"
+
 # install dependencies
 apt-get update
 apt-get -y --fix-missing install git
@@ -13,7 +18,8 @@ fi;
 
 if [ "qp-qiime2" != "$PLUGIN" ]; then
     # clone plugin repository
-    git clone https://github.com/qiita-spots/${PLUGIN}
+    git clone -b ${branch} https://github.com/${fork}/${PLUGIN};
+    echo "Clone from '${fork}', branch '${branch}'" 1>&2;
 fi;
 
 # NOTE: client api reset only works when communicating with Qitta Master,
@@ -43,6 +49,9 @@ done
 export QIITA_PORT=21174
 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE
 
+# either (old) plugins ignore this at all OR adapted plugins switch to https file transfer
+export QIITA_PLUGINCOUPLING=https
+
 # change into plugin source directory and execute actual tests
 if [ "qp-qiime2" == "$PLUGIN" ]; then
     source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest;

compose.yaml

@@ -281,7 +281,7 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita_data
+      #- qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref
       #- ./src/qtp-biom:/qtp-biom:U

compose_github.yaml

@@ -193,7 +193,7 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita_data
+      #- qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - test_tmp_dir:/tmp
     environment:
@@ -215,7 +215,7 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita_data
+      #- qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - test_tmp_dir:/tmp
       - ./references/qp-target-gene:/databases/gg/13_8/rep_set
@@ -261,7 +261,7 @@ services:
     command: ['./start_qp-deblur.sh']
     restart: no
     volumes:
-      - qiita-data:/qiita_data
+      # - qiita-data:/qiita_data
       - ./references/qiita_server_certificates:/qiita_server_certificates
       - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref
       - test_tmp_dir:/tmp
@@ -349,4 +349,4 @@ volumes:
   server-plugin-configs: # a shared directory on qiita server side (master and workers) to collect qiita plugin configuration files
   qiita-logs:
   test_tmp_dir: # many of plugin tests are written under the assumption that they are executed on same machine as qiita main and thus share /tmp - which is not the case in our docker compose scenario!
-
+