Merge pull request #148 from tattle-made/hotfix

Hotfix
tattle-made · Mar 11, 2024 · 1001e92 · 1001e92
2 parents adf12d0 + 0f7e440
commit 1001e92
Show file tree

Hide file tree

Showing 16 changed files with 253 additions and 122 deletions.
diff --git a/.github/workflows/docker-push-vidvec-benchmark-test.yml b/.github/workflows/docker-push-vidvec-benchmark-test.yml
@@ -36,7 +36,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: ./src/
-          file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton
+          file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton
           platforms: linux/arm64
           push: true
           tags: tattletech/feluda-operator-vidvec:benchmark-arm64-latest-test
diff --git a/.github/workflows/docker-push-vidvec-benchmark.yml b/.github/workflows/docker-push-vidvec-benchmark.yml
@@ -37,7 +37,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-vidvec
           workdir: src/
-          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton
+          dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton
           tags: benchmark-arm64-latest
           platforms: linux/arm64
 

diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml
@@ -67,7 +67,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-vidvec
           workdir: src/
-          dockerfile: worker/vidvec/Dockerfile.video_worker.graviton
+          dockerfile: worker/vidvec/Dockerfile.video_worker_graviton
           tags: worker-arm64-${{ steps.release.outputs.tag }}
           platforms: linux/arm64
 
@@ -91,7 +91,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
           name: tattletech/feluda-operator-audiovec
           workdir: src/
-          dockerfile: worker/audiovec/Dockerfile.audio_worker.graviton
+          dockerfile: worker/audiovec/Dockerfile.audio_worker_graviton
           tags: worker-arm64-${{ steps.release.outputs.tag }}
           platforms: linux/arm64
 

diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml
@@ -14,6 +14,9 @@ on:
 jobs:
   checks:
     if: github.event.pull_request.draft == false
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     name: Run security checks
     runs-on: ubuntu-latest
     steps:
@@ -64,3 +67,21 @@ jobs:
           local: true
           inputs: |
             ./src/core/operators/vid_vec_rep_resnet_requirements.txt
+
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          ignore-unfixed: true
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          limit-severities-for-sarif: true
+          severity: 'HIGH,CRITICAL'
+          scanners: 'vuln,config,secret'
+          skip-dirs: '.vscode,docs'
+          exit-code: '1'
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
diff --git a/src/Dockerfile b/src/Dockerfile
@@ -1,6 +1,7 @@
 # FROM jrottenberg/ffmpeg:4.0-scratch AS ffmpeg
 
 FROM python:3.11-slim-bullseye AS base
+
 # COPY --from=ffmpeg / /
 RUN apt-get update \
     && apt-get -y upgrade \
@@ -11,18 +12,27 @@ RUN apt-get update \
 #    gcc build-essential \
 #    libgl1-mesa-glx libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/* 
-ENV PATH=/root/.local/bin:$PATH
+
+# Set python user
+RUN groupadd -g 999 python && \
+    useradd --create-home -r -u 999 -g python python
+RUN mkdir /usr/app && chown python:python /usr/app
+WORKDIR /usr/app
+
+# Set venv
+RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
+ENV PATH="/usr/app/venv/bin:$PATH"
+
 RUN pip install --no-cache-dir --upgrade pip
-RUN apt-get update && apt-get -y upgrade && apt-get install -y vim curl
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends vim curl
 # RUN apt-get install -y ffmpeg
 # RUN apt-get update && \
 #     apt-get -y upgrade && \
 #     apt-get install -y tesseract-ocr tesseract-ocr-hin
-RUN apt-get update && apt-get -y upgrade && apt-get install wget
-WORKDIR /app
-COPY requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir --user -r requirements.txt
-COPY . /app
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends wget
+COPY --chown=python:python requirements.txt /usr/app/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY --chown=python:python . /usr/app
 EXPOSE 7000
 
 # RUN apt-get update \
@@ -34,11 +44,13 @@ EXPOSE 7000
 
 #### DEBUG IMAGE ####
 FROM base AS debug
-RUN apt-get update && apt-get install -y vim zsh jq
+RUN apt-get update && apt-get install -y --no-install-recommends vim zsh jq
 RUN pip install --no-cache-dir debugpy nose2
 RUN export FLASK_DEBUG=1
+USER 999
 CMD python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m flask run -h 0.0.0.0 -p 5000
 
 #### PROD IMAGE ####
-FROM base as prod
+FROM base AS prod
+USER 999
 CMD flask run --host=0.0.0.0
diff --git a/src/Dockerfile.test b/src/Dockerfile.test
@@ -1,18 +1,28 @@
 FROM python:3.11-slim-bullseye AS base
+
 RUN apt-get update \
     && apt-get -y upgrade \
     && rm -rf /var/lib/apt/lists/*
-ENV PATH=/root/.local/bin:$PATH
+
+# Set python user
+RUN groupadd -g 999 python && \
+    useradd --create-home -r -u 999 -g python python
+RUN mkdir /usr/app && chown python:python /usr/app
+WORKDIR /usr/app
+
+# Set venv
+RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
+ENV PATH="/usr/app/venv/bin:$PATH"
+
 RUN pip install --no-cache-dir --upgrade pip
-RUN apt-get update && apt-get -y upgrade && apt-get install -y wget curl grep
-WORKDIR /app
-COPY requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir --user -r requirements.txt
-COPY . /app
+RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends wget curl grep
+COPY --chown=python:python requirements.txt /usr/app/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+COPY --chown=python:python . /usr/app
 
 #### TEST IMAGE ####
 FROM base AS test
 RUN cd core/operators \
-    && pip install --no-cache-dir --user -r vid_vec_rep_resnet_requirements.txt \
-    && pip install --no-cache-dir --user -r audio_vec_embedding_requirements.txt
-
+    && pip install --no-cache-dir -r vid_vec_rep_resnet_requirements.txt \
+    && pip install --no-cache-dir -r audio_vec_embedding_requirements.txt
+USER 999
diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding
@@ -1,30 +1,39 @@
 FROM python:3.11-slim-bullseye AS base
+
 RUN apt-get update \
     && apt-get -y upgrade \
     && apt-get install -y \
     --no-install-recommends gcc build-essential \
     --no-install-recommends libgl1-mesa-glx libglib2.0-0 \
     --no-install-recommends python3-dev
-ENV PATH=/root/.local/bin:$PATH
+
+# Set python user
+RUN groupadd -g 999 python && \
+    useradd --create-home -r -u 999 -g python python
+RUN mkdir /usr/app && chown python:python /usr/app
+WORKDIR /usr/app
+
+# Set venv
+RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
+ENV PATH="/usr/app/venv/bin:$PATH"
 
 RUN pip install --no-cache-dir --upgrade pip
-WORKDIR /app
 
 # audio requirments file
-COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt
-RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt
+COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt
+RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt
 # audio vec file
-COPY ./core/operators/audio_vec_embedding.py /app/core/operators/audio_vec_embedding.py
+COPY --chown=python:python ./core/operators/audio_vec_embedding.py /usr/app/core/operators/audio_vec_embedding.py
 # audio vec test and media files
-COPY ./core/operators/sample_data/audio.wav /app/core/operators/sample_data/audio.wav
-COPY ./core/operators/test_audio_vec_embedding.py /app/core/operators/test_audio_vec_embedding.py
+COPY --chown=python:python ./core/operators/sample_data/audio.wav /usr/app/core/operators/sample_data/audio.wav
+COPY --chown=python:python ./core/operators/test_audio_vec_embedding.py /usr/app/core/operators/test_audio_vec_embedding.py
 # audio cnn model folder
-COPY ./core/operators/audio_cnn_model/ /app/core/operators/audio_cnn_model/
+COPY --chown=python:python ./core/operators/audio_cnn_model/ /usr/app/core/operators/audio_cnn_model/
 # benchmark files
-COPY ./benchmark/audiovec/ /app/benchmark/audiovec/
+COPY --chown=python:python ./benchmark/audiovec/ /usr/app/benchmark/audiovec/
 RUN chmod +x ./benchmark/audiovec/*.sh
 # main benchmark file
-COPY ./benchmark/benchmark-audio.sh /app/benchmark/benchmark-audio.sh
+COPY --chown=python:python ./benchmark/benchmark-audio.sh /usr/app/benchmark/benchmark-audio.sh
 RUN chmod +x ./benchmark/benchmark-audio.sh
 
 RUN apt-get purge -y --auto-remove \
@@ -33,7 +42,8 @@ RUN apt-get purge -y --auto-remove \
     python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
-RUN apt-get update && apt-get install -y vim zsh
-RUN apt-get update && apt-get install -y wget
+RUN apt-get update && apt-get install -y --no-install-recommends vim zsh
+RUN apt-get update && apt-get install -y --no-install-recommends wget
 
+USER 999
 CMD tail -f /dev/null
diff --git a/...c/Dockerfile.audio_vec_embedding.graviton → ...c/Dockerfile.audio_vec_embedding_graviton b/...c/Dockerfile.audio_vec_embedding.graviton → ...c/Dockerfile.audio_vec_embedding_graviton
@@ -1,18 +1,27 @@
 FROM python:3.11-slim-bullseye AS base
+
 RUN apt-get update \
     && apt-get -y upgrade \
     && apt-get install -y \
     --no-install-recommends gcc build-essential \
     --no-install-recommends libgl1-mesa-glx libglib2.0-0 \
     --no-install-recommends python3-dev
-ENV PATH=/root/.local/bin:$PATH
+
+# Set python user
+RUN groupadd -g 999 python && \
+    useradd --create-home -r -u 999 -g python python
+RUN mkdir /usr/app && chown python:python /usr/app
+WORKDIR /usr/app
+
+# Set venv
+RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
+ENV PATH="/usr/app/venv/bin:$PATH"
 
 RUN pip install --no-cache-dir --upgrade pip
-WORKDIR /app
 
 # audio requirments file
-COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt
-RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt
+COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt
+RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt
 
 ### AWS Graviton Optimization ###
 
@@ -38,17 +47,17 @@ ENV OMP_PLACES=cores
 ###
 
 # audio vec file
-COPY ./core/operators/audio_vec_embedding.py /app/core/operators/audio_vec_embedding.py
+COPY --chown=python:python ./core/operators/audio_vec_embedding.py /usr/app/core/operators/audio_vec_embedding.py
 # audio vec test and media files
-COPY ./core/operators/sample_data/audio.wav /app/core/operators/sample_data/audio.wav
-COPY ./core/operators/test_audio_vec_embedding.py /app/core/operators/test_audio_vec_embedding.py
+COPY --chown=python:python ./core/operators/sample_data/audio.wav /usr/app/core/operators/sample_data/audio.wav
+COPY --chown=python:python ./core/operators/test_audio_vec_embedding.py /usr/app/core/operators/test_audio_vec_embedding.py
 # audio cnn model folder
-COPY ./core/operators/audio_cnn_model/ /app/core/operators/audio_cnn_model/
+COPY --chown=python:python ./core/operators/audio_cnn_model/ /usr/app/core/operators/audio_cnn_model/
 # benchmark files
-COPY ./benchmark/audiovec/ /app/benchmark/audiovec/
+COPY --chown=python:python ./benchmark/audiovec/ /usr/app/benchmark/audiovec/
 RUN chmod +x ./benchmark/audiovec/*.sh
 # main benchmark file
-COPY ./benchmark/benchmark-audio.sh /app/benchmark/benchmark-audio.sh
+COPY --chown=python:python ./benchmark/benchmark-audio.sh /usr/app/benchmark/benchmark-audio.sh
 RUN chmod +x ./benchmark/benchmark-audio.sh
 
 RUN apt-get purge -y --auto-remove \
@@ -57,7 +66,8 @@ RUN apt-get purge -y --auto-remove \
     python3-dev \
     && rm -rf /var/lib/apt/lists/*
 
-RUN apt-get update && apt-get install -y vim zsh
-RUN apt-get update && apt-get install -y wget
+RUN apt-get update && apt-get install -y --no-install-recommends vim zsh
+RUN apt-get update && apt-get install -y --no-install-recommends wget
 
+USER 999
 CMD tail -f /dev/null
diff --git a/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet b/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet
@@ -1,4 +1,5 @@
 FROM python:3.11-slim@sha256:637774748f62b832dc11e7b286e48cd716727ed04b45a0322776c01bc526afc3 AS base
+
 RUN apt-get update \
     && apt-get -y upgrade \
     && apt-get install -y \
@@ -8,22 +9,31 @@ RUN apt-get update \
     gcc build-essential \
     libgl1-mesa-glx libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/* 
-ENV PATH=/root/.local/bin:$PATH
+
+# Set python user
+RUN groupadd -g 999 python && \
+    useradd --create-home -r -u 999 -g python python
+RUN mkdir /usr/app && chown python:python /usr/app
+WORKDIR /usr/app
+
+# Set venv
+RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv
+ENV PATH="/usr/app/venv/bin:$PATH"
 
 RUN pip install --no-cache-dir --upgrade pip
-WORKDIR /app
-COPY ./core/operators/image_vec_rep_resnet_requirements.txt /app/core/operators/image_vec_rep_resnet_requirements.txt
-RUN pip install --no-cache-dir --user -r /app/core/operators/image_vec_rep_resnet_requirements.txt
-COPY ./core/operators/image_vec_rep_resnet.py /app/core/operators/image_vec_rep_resnet.py
+COPY --chown=python:python ./core/operators/image_vec_rep_resnet_requirements.txt /usr/app/core/operators/image_vec_rep_resnet_requirements.txt
+RUN pip install --no-cache-dir -r /usr/app/core/operators/image_vec_rep_resnet_requirements.txt
+COPY --chown=python:python ./core/operators/image_vec_rep_resnet.py /usr/app/core/operators/image_vec_rep_resnet.py
 
-COPY ./core/operators/sample_data/text.png /app/core/operators/sample_data/text.png
-COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py
-COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py
+COPY --chown=python:python ./core/operators/sample_data/text.png /usr/app/core/operators/sample_data/text.png
+COPY --chown=python:python ./core/operators/test_image_vec_rep_resnet.py /usr/app/core/operators/test_image_vec_rep_resnet.py
+COPY --chown=python:python ./core/operators/test_image_vec_rep_resnet.py /usr/app/core/operators/test_image_vec_rep_resnet.py
 
-COPY ./image_vec_operator_profile.py /app/image_vec_operator_profile.py
-COPY ./image_vec_operator_profile_memray.sh /app/image_vec_operator_profile_memray.sh
-COPY ./image_vec_operator_profile_pyinstrument.sh /app/image_vec_operator_profile_pyinstrument.sh
+COPY --chown=python:python ./image_vec_operator_profile.py /usr/app/image_vec_operator_profile.py
+COPY --chown=python:python ./image_vec_operator_profile_memray.sh /usr/app/image_vec_operator_profile_memray.sh
+COPY --chown=python:python ./image_vec_operator_profile_pyinstrument.sh /usr/app/image_vec_operator_profile_pyinstrument.sh
 RUN chmod +x image_vec_operator_profile_memray.sh
 RUN chmod +x image_vec_operator_profile_pyinstrument.sh
 
+USER 999
 CMD tail -f /dev/null