Tinqiita: qiita in a docker compose setup

.github/workflows/buildContainer.yaml

@@ -0,0 +1,297 @@
+# todo: "make" files once and push around through artifacts!
+
+name: docker
+
+on:
+  pull_request:
+    branches:
+      - tinqiita
+  push:
+    branches:
+      - tinqiita
+
+jobs:
+  make_certificates:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Create certificate
+        # second copy of "qiita_server_certificates" is necessary to match path for docker build, first copy for mounting into container
+        run: |
+          make ./references/qiita_server_certificates ./environments/qiita_db.env ./environments/qiita.env config
+          cp -r ./references/qiita_server_certificates ./qiita_server_certificates
+
+      - name: Store certifactes for follow up jobs
+        uses: actions/upload-artifact@v4
+        with:
+          name: certificates
+          path: |
+            ./qiita_server_certificates
+            ./references/qiita_server_certificates
+
+  build_main:
+    needs: make_certificates
+    strategy:
+      matrix:
+        container: ["nginx", "qiita", "plugin_collector"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+          cp Images/${{ matrix.container }}/start_${{ matrix.container }}.sh Images/test_plugin.sh Images/nginx/nginx_qiita.conf Images/${{ matrix.container }}/requirements.txt .
+          cp Images/qiita/config_portal.cfg Images/qiita/config_qiita_oidc.cfg Images/qiita/drop_workflows.py Images/qiita/start_plugin.py Images/qiita/start_qiita-initDB.sh Images/qiita/start_qiita.sh .
+          cp Images/plugin_collector/collect_configs.py Images/plugin_collector/fix_test_db.py Images/plugin_collector/stefan_cert.conf Images/plugin_collector/stefan_csr.conf .
+
+      - name: Download certificates from job build_main
+        uses: actions/download-artifact@v4
+        with:
+          name: certificates
+          path: ./          
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build main qiita images and push to github's own registry
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          file: Images/${{ matrix.container }}/${{ matrix.container }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.container }}:testcandidate
+          cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+          cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
+
+  make_references:
+    needs: build_main
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Create partially fake reference databases
+        run: |
+          mkdir -p ./references/qp-deblur/ ./references/qp-target-gene
+          make ./environments/qiita_db.env ./environments/qiita.env config ./references/qp-deblur/reference-gg-raxml-bl.tre
+          for f in `echo "references/qp-target-gene/97_otus.fasta references/qp-target-gene/97_otus.tree references/qp-target-gene/97_otu_taxonomy.txt"`; do echo "fake" > $f; done
+
+      - name: Store fake references for follow up jobs
+        uses: actions/upload-artifact@v4
+        with:
+          name: fake_references
+          path: |
+            ./references/qp-deblur
+            ./references/qp-target-gene
+            ./environments
+
+  build_plugins:
+    needs:
+      - build_main
+      - make_references
+    strategy:
+      matrix:
+        plugin: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+
+      - name: Download certificates from job build_main
+        uses: actions/download-artifact@v4
+        with:
+          name: certificates
+          path: ./
+      - name: Download fake references
+        uses: actions/download-artifact@v4
+        with:
+          name: fake_references
+          path: ./
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build plugin images
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+          cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+          cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
+
+      - name: adapt compose file to select specific plugin
+        run: |
+          sed -i "s/MATRIXPLUGIN/${{ matrix.plugin }}/g" compose_github.yaml
+
+      - name: Run docker compose
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: "compose_github.yaml"
+          services: |
+            nginx
+
+      - name: Execute tests in the running services
+        run: |
+          sleep 5
+          docker compose exec ${{ matrix.plugin }} /bin/bash -c "bash /test_plugin.sh"
+
+      - name: Push image to ghcr (only if tests passed)
+        if: success()
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          file: Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.plugin }}:testcandidate
+
+  # the qp-qiime2 plugin cannot be tested in isolation, it also needs qtp-diversity and qtp-visualization to be active in qiita
+  build_mulit_plugins:
+    needs:
+      - build_plugins
+    strategy:
+      matrix:
+        multiplugin: ["qp-qiime2"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+          cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+
+      - name: Download certificates from job build_main
+        uses: actions/download-artifact@v4
+        with:
+          name: certificates
+          path: ./
+      - name: Download fake references
+        uses: actions/download-artifact@v4
+        with:
+          name: fake_references
+          path: ./
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build plugin images
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          load: true
+          file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
+          cache-from: type=gha,scope=tinqiita-${{ github.ref_name }}
+          cache-to: type=gha,scope=tinqiita-${{ github.ref_name }},mode=max
+
+      - name: adapt compose file to select specific plugin
+        run: |
+          if [[ "${{ matrix.multiplugin }}" == "qp-qiime2" ]]; then sed -i 's|- QIITA_PLUGINS="MATRIXPLUGIN:"|- QIITA_PLUGINS="${{ matrix.multiplugin }}:qtp-diversity:qtp-visualization:"|' compose_github.yaml; sed -i 's|MATRIXPLUGIN:|${{ matrix.multiplugin }}:\n        condition: service_started\n      qtp-diversity:\n        condition: service_started\n      qtp-visualization:|' compose_github.yaml; fi;
+
+      - name: Run docker compose
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: "compose_github.yaml"
+          services: |
+            nginx
+
+      - name: Execute tests in the running services
+        run: |
+          sleep 5
+          docker compose exec ${{ matrix.multiplugin }} /bin/bash -c "bash /test_plugin.sh"
+
+      - name: Push image to ghcr (only if tests passed)
+        if: success()
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          file: Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.multiplugin }}:testcandidate
+
+  publish_images:
+    needs:
+      - build_plugins
+      - build_main
+      - build_mulit_plugins
+    strategy:
+      matrix:
+        image: ["qp-deblur", "qtp-biom", "qtp-sequencing", "qtp-visualization", "qtp-diversity", "qp-target-gene", "qtp-job-output-folder",
+                 "nginx", "qiita", "plugin_collector",
+                 "qp-qiime2"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout tinqiita repo
+        uses: actions/checkout@v4
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to computational.bio registry
+        uses: docker/login-action@v3
+        with:
+          registry: harbor.computational.bio.uni-giessen.de
+          username: ${{ vars.HARBOR_CB_USERNAME }}
+          password: ${{ secrets.HARBOR_CB_SECRET }}
+
+      - name: Pull image from GHCR
+        run: docker pull ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate
+
+      - name: Read image version from dockerfile
+        id: vars
+        run: |
+          VERSION=$(head -n 1 Images/${{ matrix.image }}/${{ matrix.image }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
+          echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
+
+      - name: Retag image for Docker Hub
+        run: |
+          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+          docker tag ghcr.io/${{ github.repository }}/${{ matrix.image }}:testcandidate harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest
+
+      - name: Push image to Docker Hub
+        run: |
+          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:${{ env.IMAGE_TAG }}
+          docker push harbor.computational.bio.uni-giessen.de/tinqiita/${{ matrix.image }}:latest

.gitignore

@@ -1,2 +1,5 @@
-environments/db.env
-environments/keycloak.env
+environments/*.env
+logs/*
+.built_image_*
+Certificates/*
+references/*