diff --git a/.github/workflows/buildContainer.yaml b/.github/workflows/buildContainer.yaml index 737c167..6a7065e 100644 --- a/.github/workflows/buildContainer.yaml +++ b/.github/workflows/buildContainer.yaml @@ -162,7 +162,9 @@ jobs: - name: Execute tests in the running services run: | sleep 5 - docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" + fork=`if [[ "${{ matrix.plugin }}" == "qp-deblur" || "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "jlab"; else echo "qiita-spots"; fi` + branch=`if [[ "${{ matrix.plugin }}" == "qp-deblur" ]]; then echo "uncouplePlugins"; elif [[ "${{ matrix.plugin }}" == "qp-target-gene" ]]; then echo "uncouplePlugin"; elif [[ "${{ matrix.plugin }}" == "qtp-job-output-folder" ]]; then echo "main"; else echo "master"; fi` + docker compose exec ${{ matrix.plugin }} /bin/bash -c "PLUGIN_FORK=$fork PLUGIN_BRANCH=$branch bash /test_plugin.sh" - name: Push image to ghcr (only if tests passed) if: success() diff --git a/Images/qiita/qiita.dockerfile b/Images/qiita/qiita.dockerfile index 0a1ab71..2f7aab2 100644 --- a/Images/qiita/qiita.dockerfile +++ b/Images/qiita/qiita.dockerfile @@ -47,9 +47,14 @@ RUN pip install \ psycopg2-binary -# Clone the Qiita Repo +# Clone the Qiita Repo: currently we need the oidc changes from our jlab fork + changes in the tornado_FetchFileFromCentralHandler branch, which send files if requested directly from tornado instead of nginx (happens in testing) # RUN git clone -b master https://github.com/qiita-spots/qiita.git -RUN git clone -b auth_oidc https://github.com/jlab/qiita.git +RUN git clone -b auth_oidc https://github.com/jlab/qiita.git \ + && cd qiita \ + && git config pull.rebase false \ + && git config --global user.email "jlab@uni-giessen.de" \ + && git config --global user.name "Stefan" && \ + git pull origin tornado_FetchFileFromCentralHandler # should tests re-populate the DB, ensure private plugin, qtp-biom and qp-target-gene use the correct conda env RUN sed -i "s|'source /home/runner/.profile; conda activate qiita'|'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita'|" /qiita/qiita_db/support_files/populate_test_db.sql diff --git a/Images/qp-deblur/qp-deblur.dockerfile b/Images/qp-deblur/qp-deblur.dockerfile index 067fe51..186e67e 100644 --- a/Images/qp-deblur/qp-deblur.dockerfile +++ b/Images/qp-deblur/qp-deblur.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.22 +# VERSION: 2025.09.12 # ========================== # Stage 1: Build wheels @@ -37,7 +37,8 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/deblur", "/bin/bash", "-c"] ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +# RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/" qiita_client/qiita_client/plugin.py RUN cd qiita_client && pip install --no-cache-dir . @@ -130,7 +131,7 @@ RUN cd qiita_client && pip install . RUN mkdir -p /qiita_server_certificates/ COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ -RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` filesystem +RUN /usr/local/bin/configure_deblur --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-deblur/" /unshared_plugins/*.conf # remove conda command from tigger.py diff --git a/Images/qp-target-gene/qp-target-gene.dockerfile b/Images/qp-target-gene/qp-target-gene.dockerfile index dc3b9f7..5482552 100644 --- a/Images/qp-target-gene/qp-target-gene.dockerfile +++ b/Images/qp-target-gene/qp-target-gene.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.08.28 +# VERSION: 2025.09.17 # ========================== # Stage 1: Build wheels @@ -44,11 +44,13 @@ RUN python2.7 get-pip2.7.py --force-reinstall RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git + RUN cd /qiita_client && pip install --no-cache-dir . RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip -RUN git clone https://github.com/qiita-spots/qp-target-gene.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qp-target-gene.git WORKDIR /qp-target-gene RUN pip install biom-format RUN pip install -e . @@ -127,7 +129,7 @@ RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt COPY qiita_server_certificates/*_server.* /qiita_server_certificates/ RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/configure_target_gene RUN sed -i "s|^#\!.*|#\!/usr/bin/python2|" /usr/local/bin/start_target_gene -RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` +RUN configure_target_gene --env-script "true" --server-cert `find /qiita_server_certificates/ -name "*_server.crt" -type f` https RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-target-gene/" /unshared_plugins/*.conf # for testing diff --git a/Images/qtp-sequencing/qtp-sequencing.dockerfile b/Images/qtp-sequencing/qtp-sequencing.dockerfile index 20cbd1d..06da58b 100644 --- a/Images/qtp-sequencing/qtp-sequencing.dockerfile +++ b/Images/qtp-sequencing/qtp-sequencing.dockerfile @@ -1,4 +1,4 @@ -# VERSION: 2025.09.08 +# VERSION: 2025.09.29 # ========================== # Stage 1: Build wheels (~5.8 GB) @@ -36,14 +36,17 @@ SHELL ["conda", "run", "-p", "/opt/conda/envs/qtp-sequencing", "/bin/bash", "-c" RUN pip install -U pip #RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip -RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +#RUN git clone -b master https://github.com/qiita-spots/qiita_client.git +RUN git clone -b enable_pluginprotocol_change https://github.com/jlab/qiita_client.git + RUN cd qiita_client && pip install --no-cache-dir . # RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip RUN git clone -b master https://github.com/qiita-spots/qiita-files.git RUN cd /qiita-files && pip install -e . -v -RUN git clone https://github.com/qiita-spots/qtp-sequencing.git +#RUN git clone https://github.com/qiita-spots/qtp-sequencing.git +RUN git clone -b uncouplePlugin https://github.com/jlab/qtp-sequencing.git WORKDIR /qtp-sequencing RUN sed -i "s|'qiita-files @ https://github.com/'||" setup.py RUN sed -i "s|'qiita-spots/qiita-files/archive/master.zip',||" setup.py diff --git a/Images/test_plugin.sh b/Images/test_plugin.sh index 2f03968..82c31c3 100644 --- a/Images/test_plugin.sh +++ b/Images/test_plugin.sh @@ -2,6 +2,11 @@ echo "plugin to be tested is: '$PLUGIN'" +# by default, clone from master. But can also clone different branches +branch="${PLUGIN_BRANCH:-master}" +# by default, clone from qiita-spots branch. But can also use alternative forks +fork="${PLUGIN_FORK:-qiita-spots}" + # install dependencies apt-get update apt-get -y --fix-missing install git @@ -13,7 +18,8 @@ fi; if [ "qp-qiime2" != "$PLUGIN" ]; then # clone plugin repository - git clone https://github.com/qiita-spots/${PLUGIN} + git clone -b ${branch} https://github.com/${fork}/${PLUGIN}; + echo "Clone from '${fork}', branch '${branch}'" 1>&2; fi; # NOTE: client api reset only works when communicating with Qitta Master, @@ -43,6 +49,9 @@ done export QIITA_PORT=21174 export QIITA_ROOTCA_CERT=$SSL_CERT_FILE +# either (old) plugins ignore this at all OR adapted plugins switch to https file transfer +export QIITA_PLUGINCOUPLING=https + # change into plugin source directory and execute actual tests if [ "qp-qiime2" == "$PLUGIN" ]; then source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiime2; cd ${PLUGIN} && pytest; diff --git a/compose.yaml b/compose.yaml index 53a4b4d..bffee2b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -281,7 +281,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref #- ./src/qtp-biom:/qtp-biom:U diff --git a/compose_github.yaml b/compose_github.yaml index 443a6f9..954803f 100644 --- a/compose_github.yaml +++ b/compose_github.yaml @@ -193,7 +193,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp environment: @@ -215,7 +215,7 @@ services: # tty: true restart: no volumes: - - qiita-data:/qiita_data + #- qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - test_tmp_dir:/tmp - ./references/qp-target-gene:/databases/gg/13_8/rep_set @@ -261,7 +261,7 @@ services: command: ['./start_qp-deblur.sh'] restart: no volumes: - - qiita-data:/qiita_data + # - qiita-data:/qiita_data - ./references/qiita_server_certificates:/qiita_server_certificates - ./references/qp-deblur:/opt/conda/envs/deblur/share/fragment-insertion/ref - test_tmp_dir:/tmp @@ -349,4 +349,4 @@ volumes: server-plugin-configs: # a shared directory on qiita server side (master and workers) to collect qiita plugin configuration files qiita-logs: test_tmp_dir: # many of plugin tests are written under the assumption that they are executed on same machine as qiita main and thus share /tmp - which is not the case in our docker compose scenario! - \ No newline at end of file +