Add multi plugin #114
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # todo: "make" files once and push around through artifacts! | |
| name: docker | |
| on: | |
| pull_request: | |
| branches: | |
| - tinqiita | |
| push: | |
| branches: | |
| - tinqiita | |
| jobs: | |
| make_certificates: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Create certificate | |
| # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container | |
| run: | | |
| make ./references/qiita_server_certificates ./environments/qiita_db.env ./environments/qiita.env config | |
| cp -r ./references/qiita_server_certificates ./qiita_server_certificates | |
| - name: Store certifactes for follow up jobs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: certificates | |
| path: | | |
| ./qiita_server_certificates | |
| ./references/qiita_server_certificates | |
| build_main: | |
| needs: make_certificates | |
| strategy: | |
| matrix: | |
| container: ["nginx", "qiita", "plugin_collector"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Read image version from dockerfile | |
| id: vars | |
| run: | | |
| VERSION=$(head -n 1 Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | cut -d ":" -f 2- | tr -d " ") | |
| echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV | |
| cp Images/${{ matrix.container }}/start_${{ matrix.container }}.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt . | |
| cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh . | |
| cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf . | |
| - name: Download certificates from job build_main | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: certificates | |
| path: ./ | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build main qiita images and push to github's own registry | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| file: Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.container }}:testcandidate | |
| cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} | |
| cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max | |
| make_references: | |
| needs: build_main | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Create partially fake reference databases | |
| run: | | |
| mkdir -p ./references/qp-deblur/ ./references/qp-target-gene | |
| make ./environments/qiita_db.env ./environments/qiita.env config ./references/qp-deblur/reference-gg-raxml-bl.tre | |
| for f in `echo "references/qp-target-gene/97_otus.fasta references/qp-target-gene/97_otus.tree references/qp-target-gene/97_otu_taxonomy.txt"`; do echo "fake" > $f; done | |
| - name: Store fake references for follow up jobs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fake_references | |
| path: | | |
| ./references/qp-deblur | |
| ./references/qp-target-gene | |
| ./environments | |
| build_plugins: | |
| needs: | |
| - build_main | |
| - make_references | |
| strategy: | |
| matrix: | |
| plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Read image version from dockerfile | |
| id: vars | |
| run: | | |
| VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") | |
| echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV | |
| cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . | |
| - name: Download certificates from job build_main | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: certificates | |
| path: ./ | |
| - name: Download fake references | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: fake_references | |
| path: ./ | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build plugin images | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| load: true | |
| file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate | |
| cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} | |
| cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max | |
| - name: adapt compose file to select specific plugin | |
| run: | | |
| sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml | |
| - name: Run docker compose | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: "compose_github.yaml" | |
| services: | | |
| nginx | |
| - name: Execute tests in the running services | |
| run: | | |
| sleep 5 | |
| docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh" | |
| - name: Push image to ghcr (only if tests passed) | |
| if: success() | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate | |
| # the qp-qiime2 plugin cannot be tested in isolation, it also needs qtp-diversity and qtp-visualization to be active in qiita | |
| build_mulit_plugins: | |
| needs: | |
| - build_plugins | |
| strategy: | |
| matrix: | |
| multiplugin: ["qp-qiime2"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Read image version from dockerfile | |
| id: vars | |
| run: | | |
| VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ") | |
| echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV | |
| cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py . | |
| - name: Download certificates from job build_main | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: certificates | |
| path: ./ | |
| - name: Download fake references | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: fake_references | |
| path: ./ | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build plugin images | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| load: true | |
| file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate | |
| cache-from: type=gha,scope=tinqiita-${{ github.ref_name }} | |
| cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max | |
| - name: adapt compose file to select specific plugin | |
| run: | | |
| if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n condition: service_started\n qtp-diversity:\n condition: service_started\n qtp-visualization:|' compose_github.yaml; fi; | |
| - name: Run docker compose | |
| uses: hoverkraft-tech/[email protected] | |
| with: | |
| compose-file: "compose_github.yaml" | |
| services: | | |
| nginx | |
| - name: Execute tests in the running services | |
| run: | | |
| sleep 5 | |
| docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh" | |
| - name: Push image to ghcr (only if tests passed) | |
| if: success() | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | |
| tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate | |
| publish_images: | |
| needs: | |
| - build_plugins | |
| - build_main | |
| - build_mulit_plugins | |
| strategy: | |
| matrix: | |
| image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder", | |
| "nginx", "qiita", "plugin_collector", | |
| "qp-qiime2"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout tinqiita repo | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to computational.bio registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: harbor.computational.bio.uni-giessen.de | |
| username: ${{ vars.HARBOR_CB_USERNAME }} | |
| password: ${{ secrets.HARBOR_CB_SECRET }} | |
| - name: Pull image from GHCR | |
| run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate | |
| - name: Read image version from dockerfile | |
| id: vars | |
| run: | | |
| VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ") | |
| echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV | |
| - name: Retag image for Docker Hub | |
| run: | | |
| docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }} | |
| docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest | |
| - name: Push image to Docker Hub | |
| run: | | |
| docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }} | |
| docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest |