From a760d2ee4626b9094607470de6f3bf2ebcd3cb5e Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:50:44 +0530 Subject: [PATCH 1/9] fix (security): Harden dockerfiles - Added unprivileged python user - Created venv for all pip install - Added chown for all files and dirs - Added --no-install-recommends to apt-get install cmd - Fixed feluda core server.py import issue --- src/Dockerfile | 30 +++++++++++----- src/Dockerfile.test | 28 ++++++++++----- .../audiovec/Dockerfile.audio_vec_embedding | 34 +++++++++++------- .../Dockerfile.audio_vec_embedding.graviton | 34 +++++++++++------- .../imgvec/Dockerfile.image_vec_rep_resnet | 32 +++++++++++------ .../vidvec/Dockerfile.vid_vec_rep_resnet | 32 +++++++++++------ .../Dockerfile.vid_vec_rep_resnet.graviton | 35 +++++++++++-------- src/server.py | 5 +-- src/worker/audiovec/Dockerfile.audio_worker | 30 ++++++++++------ .../audiovec/Dockerfile.audio_worker.graviton | 30 ++++++++++------ src/worker/vidvec/Dockerfile.video_worker | 28 ++++++++++----- .../vidvec/Dockerfile.video_worker.graviton | 28 ++++++++++----- 12 files changed, 228 insertions(+), 118 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 02513f34..ae11f2e0 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,6 +1,7 @@ # FROM jrottenberg/ffmpeg:4.0-scratch AS ffmpeg FROM python:3.11-slim-bullseye AS base + # COPY --from=ffmpeg / / RUN apt-get update \ && apt-get -y upgrade \ @@ -11,18 +12,27 @@ RUN apt-get update \ # gcc build-essential \ # libgl1-mesa-glx libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" + RUN pip install --no-cache-dir --upgrade pip -RUN apt-get update && apt-get -y upgrade && apt-get install -y vim curl +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends vim curl # RUN apt-get install -y ffmpeg # RUN apt-get update && \ # apt-get -y upgrade && \ # apt-get install -y tesseract-ocr tesseract-ocr-hin -RUN apt-get update && apt-get -y upgrade && apt-get install wget -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt -COPY . /app +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends wget +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt +COPY --chown=python:python . /usr/app EXPOSE 7000 # RUN apt-get update \ @@ -34,11 +44,13 @@ EXPOSE 7000 #### DEBUG IMAGE #### FROM base AS debug -RUN apt-get update && apt-get install -y vim zsh jq +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh jq RUN pip install --no-cache-dir debugpy nose2 RUN export FLASK_DEBUG=1 +USER 999 CMD python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m flask run -h 0.0.0.0 -p 5000 #### PROD IMAGE #### -FROM base as prod +FROM base AS prod +USER 999 CMD flask run --host=0.0.0.0 diff --git a/src/Dockerfile.test b/src/Dockerfile.test index 7cfa091f..7b71bf95 100644 --- a/src/Dockerfile.test +++ b/src/Dockerfile.test @@ -1,18 +1,28 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && rm -rf /var/lib/apt/lists/* -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" + RUN pip install --no-cache-dir --upgrade pip -RUN apt-get update && apt-get -y upgrade && apt-get install -y wget curl grep -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt -COPY . /app +RUN apt-get update && apt-get -y upgrade && apt-get install -y --no-install-recommends wget curl grep +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt +COPY --chown=python:python . /usr/app #### TEST IMAGE #### FROM base AS test RUN cd core/operators \ - && pip install --no-cache-dir --user -r vid_vec_rep_resnet_requirements.txt \ - && pip install --no-cache-dir --user -r audio_vec_embedding_requirements.txt - + && pip install --no-cache-dir -r vid_vec_rep_resnet_requirements.txt \ + && pip install --no-cache-dir -r audio_vec_embedding_requirements.txt +USER 999 diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding index 95b5561b..296f5745 100644 --- a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding +++ b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding @@ -1,30 +1,39 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app # audio requirments file -COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt +COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt # audio vec file -COPY ./core/operators/audio_vec_embedding.py /app/core/operators/audio_vec_embedding.py +COPY --chown=python:python ./core/operators/audio_vec_embedding.py /usr/app/core/operators/audio_vec_embedding.py # audio vec test and media files -COPY ./core/operators/sample_data/audio.wav /app/core/operators/sample_data/audio.wav -COPY ./core/operators/test_audio_vec_embedding.py /app/core/operators/test_audio_vec_embedding.py +COPY --chown=python:python ./core/operators/sample_data/audio.wav /usr/app/core/operators/sample_data/audio.wav +COPY --chown=python:python ./core/operators/test_audio_vec_embedding.py /usr/app/core/operators/test_audio_vec_embedding.py # audio cnn model folder -COPY ./core/operators/audio_cnn_model/ /app/core/operators/audio_cnn_model/ +COPY --chown=python:python ./core/operators/audio_cnn_model/ /usr/app/core/operators/audio_cnn_model/ # benchmark files -COPY ./benchmark/audiovec/ /app/benchmark/audiovec/ +COPY --chown=python:python ./benchmark/audiovec/ /usr/app/benchmark/audiovec/ RUN chmod +x ./benchmark/audiovec/*.sh # main benchmark file -COPY ./benchmark/benchmark-audio.sh /app/benchmark/benchmark-audio.sh +COPY --chown=python:python ./benchmark/benchmark-audio.sh /usr/app/benchmark/benchmark-audio.sh RUN chmod +x ./benchmark/benchmark-audio.sh RUN apt-get purge -y --auto-remove \ @@ -33,7 +42,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y vim zsh -RUN apt-get update && apt-get install -y wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh +RUN apt-get update && apt-get install -y --no-install-recommends wget +USER 999 CMD tail -f /dev/null diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton index 92c81712..a43fb86d 100644 --- a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton +++ b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton @@ -1,18 +1,27 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app # audio requirments file -COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt +COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt ### AWS Graviton Optimization ### @@ -38,17 +47,17 @@ ENV OMP_PLACES=cores ### # audio vec file -COPY ./core/operators/audio_vec_embedding.py /app/core/operators/audio_vec_embedding.py +COPY --chown=python:python ./core/operators/audio_vec_embedding.py /usr/app/core/operators/audio_vec_embedding.py # audio vec test and media files -COPY ./core/operators/sample_data/audio.wav /app/core/operators/sample_data/audio.wav -COPY ./core/operators/test_audio_vec_embedding.py /app/core/operators/test_audio_vec_embedding.py +COPY --chown=python:python ./core/operators/sample_data/audio.wav /usr/app/core/operators/sample_data/audio.wav +COPY --chown=python:python ./core/operators/test_audio_vec_embedding.py /usr/app/core/operators/test_audio_vec_embedding.py # audio cnn model folder -COPY ./core/operators/audio_cnn_model/ /app/core/operators/audio_cnn_model/ +COPY --chown=python:python ./core/operators/audio_cnn_model/ /usr/app/core/operators/audio_cnn_model/ # benchmark files -COPY ./benchmark/audiovec/ /app/benchmark/audiovec/ +COPY --chown=python:python ./benchmark/audiovec/ /usr/app/benchmark/audiovec/ RUN chmod +x ./benchmark/audiovec/*.sh # main benchmark file -COPY ./benchmark/benchmark-audio.sh /app/benchmark/benchmark-audio.sh +COPY --chown=python:python ./benchmark/benchmark-audio.sh /usr/app/benchmark/benchmark-audio.sh RUN chmod +x ./benchmark/benchmark-audio.sh RUN apt-get purge -y --auto-remove \ @@ -57,7 +66,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y vim zsh -RUN apt-get update && apt-get install -y wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh +RUN apt-get update && apt-get install -y --no-install-recommends wget +USER 999 CMD tail -f /dev/null \ No newline at end of file diff --git a/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet b/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet index 78da84fb..05b8d946 100644 --- a/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet +++ b/src/benchmark/imgvec/Dockerfile.image_vec_rep_resnet @@ -1,4 +1,5 @@ FROM python:3.11-slim@sha256:637774748f62b832dc11e7b286e48cd716727ed04b45a0322776c01bc526afc3 AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ @@ -8,22 +9,31 @@ RUN apt-get update \ gcc build-essential \ libgl1-mesa-glx libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -COPY ./core/operators/image_vec_rep_resnet_requirements.txt /app/core/operators/image_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/image_vec_rep_resnet_requirements.txt -COPY ./core/operators/image_vec_rep_resnet.py /app/core/operators/image_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/image_vec_rep_resnet_requirements.txt /usr/app/core/operators/image_vec_rep_resnet_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/image_vec_rep_resnet_requirements.txt +COPY --chown=python:python ./core/operators/image_vec_rep_resnet.py /usr/app/core/operators/image_vec_rep_resnet.py -COPY ./core/operators/sample_data/text.png /app/core/operators/sample_data/text.png -COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py -COPY ./core/operators/test_image_vec_rep_resnet.py /app/core/operators/test_image_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/sample_data/text.png /usr/app/core/operators/sample_data/text.png +COPY --chown=python:python ./core/operators/test_image_vec_rep_resnet.py /usr/app/core/operators/test_image_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/test_image_vec_rep_resnet.py /usr/app/core/operators/test_image_vec_rep_resnet.py -COPY ./image_vec_operator_profile.py /app/image_vec_operator_profile.py -COPY ./image_vec_operator_profile_memray.sh /app/image_vec_operator_profile_memray.sh -COPY ./image_vec_operator_profile_pyinstrument.sh /app/image_vec_operator_profile_pyinstrument.sh +COPY --chown=python:python ./image_vec_operator_profile.py /usr/app/image_vec_operator_profile.py +COPY --chown=python:python ./image_vec_operator_profile_memray.sh /usr/app/image_vec_operator_profile_memray.sh +COPY --chown=python:python ./image_vec_operator_profile_pyinstrument.sh /usr/app/image_vec_operator_profile_pyinstrument.sh RUN chmod +x image_vec_operator_profile_memray.sh RUN chmod +x image_vec_operator_profile_pyinstrument.sh +USER 999 CMD tail -f /dev/null diff --git a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet index 1e023dca..53e3234a 100644 --- a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet +++ b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet @@ -1,29 +1,38 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app # video requirments file -COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt # video vec file -COPY ./core/operators/vid_vec_rep_resnet.py /app/core/operators/vid_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet.py /usr/app/core/operators/vid_vec_rep_resnet.py # video vec test file -COPY ./core/operators/sample_data/sample-cat-video.mp4 /app/core/operators/sample_data/sample-cat-video.mp4 -COPY ./core/operators/test_vid_vec_rep_resnet.py /app/core/operators/test_vid_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/sample_data/sample-cat-video.mp4 /usr/app/core/operators/sample_data/sample-cat-video.mp4 +COPY --chown=python:python ./core/operators/test_vid_vec_rep_resnet.py /usr/app/core/operators/test_vid_vec_rep_resnet.py # media factory files -COPY ./core/models/ /app/core/models/ +COPY --chown=python:python ./core/models/ /usr/app/core/models/ # benchmark files -COPY ./benchmark/vidvec/ /app/benchmark/vidvec/ +COPY --chown=python:python ./benchmark/vidvec/ /usr/app/benchmark/vidvec/ RUN chmod +x ./benchmark/vidvec/*.sh # main benchmark file -COPY ./benchmark/benchmark-video.sh /app/benchmark/benchmark-video.sh +COPY --chown=python:python ./benchmark/benchmark-video.sh /usr/app/benchmark/benchmark-video.sh RUN chmod +x benchmark/benchmark-video.sh RUN apt-get purge -y --auto-remove \ @@ -32,7 +41,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y vim zsh wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh wget RUN pip install numpy Pillow wget requests Werkzeug +USER 999 CMD tail -f /dev/null \ No newline at end of file diff --git a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton index 13df73ea..693866ae 100644 --- a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton +++ b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton @@ -1,17 +1,23 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -# video requirments file -COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt ### AWS Graviton Optimization ### @@ -37,20 +43,20 @@ ENV OMP_PLACES=cores ### # video requirments file -COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt # video vec file -COPY ./core/operators/vid_vec_rep_resnet.py /app/core/operators/vid_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet.py /usr/app/core/operators/vid_vec_rep_resnet.py # video vec test file -COPY ./core/operators/sample_data/sample-cat-video.mp4 /app/core/operators/sample_data/sample-cat-video.mp4 -COPY ./core/operators/test_vid_vec_rep_resnet.py /app/core/operators/test_vid_vec_rep_resnet.py +COPY --chown=python:python ./core/operators/sample_data/sample-cat-video.mp4 /usr/app/core/operators/sample_data/sample-cat-video.mp4 +COPY --chown=python:python ./core/operators/test_vid_vec_rep_resnet.py /usr/app/core/operators/test_vid_vec_rep_resnet.py # media factory files -COPY ./core/models/ /app/core/models/ +COPY --chown=python:python ./core/models/ /usr/app/core/models/ # benchmark files -COPY ./benchmark/vidvec/ /app/benchmark/vidvec/ +COPY --chown=python:python ./benchmark/vidvec/ /usr/app/benchmark/vidvec/ RUN chmod +x ./benchmark/vidvec/*.sh # main benchmark file -COPY ./benchmark/benchmark-video.sh /app/benchmark/benchmark-video.sh +COPY --chown=python:python ./benchmark/benchmark-video.sh /usr/app/benchmark/benchmark-video.sh RUN chmod +x benchmark/benchmark-video.sh RUN apt-get purge -y --auto-remove \ @@ -59,7 +65,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y vim zsh wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh wget RUN pip install numpy Pillow wget requests Werkzeug +USER 999 CMD tail -f /dev/null diff --git a/src/server.py b/src/server.py index 57486a86..ebb4d7f0 100644 --- a/src/server.py +++ b/src/server.py @@ -1,7 +1,8 @@ import logging from core.feluda import ComponentType, Feluda -from endpoint import health, index, search +from endpoint import health, search +import endpoint.index.endpoint as index_endpoint log = logging.getLogger(__name__) logging.basicConfig(level="INFO") @@ -9,7 +10,7 @@ try: feluda = Feluda("config-server.yml") feluda.set_endpoints( - [health.HealthEndpoint, index.endpoint.IndexEndpoint, search.SearchEndpoint] + [health.HealthEndpoint, index_endpoint.IndexEndpoint, search.SearchEndpoint] ) # feluda.set_endpoints([health.HealthEndpoint, index.endpoint.IndexEndpoint, search.endpoint]) # feluda.server.start() diff --git a/src/worker/audiovec/Dockerfile.audio_worker b/src/worker/audiovec/Dockerfile.audio_worker index 4464443b..13ba745d 100644 --- a/src/worker/audiovec/Dockerfile.audio_worker +++ b/src/worker/audiovec/Dockerfile.audio_worker @@ -1,22 +1,31 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH -RUN apt-get update && apt-get install -y vim zsh curl -RUN apt-get update && apt-get install -y wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh curl +RUN apt-get update && apt-get install -y --no-install-recommends wget + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # install audio operator requirements -COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt +COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt RUN apt-get purge -y --auto-remove \ gcc build-essential \ @@ -24,7 +33,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -COPY . /app +COPY --chown=python:python . /usr/app EXPOSE 7000 -FROM base AS debug \ No newline at end of file +FROM base AS debug +USER 999 diff --git a/src/worker/audiovec/Dockerfile.audio_worker.graviton b/src/worker/audiovec/Dockerfile.audio_worker.graviton index 30521710..3b75da82 100644 --- a/src/worker/audiovec/Dockerfile.audio_worker.graviton +++ b/src/worker/audiovec/Dockerfile.audio_worker.graviton @@ -1,22 +1,31 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH -RUN apt-get update && apt-get install -y vim zsh curl -RUN apt-get update && apt-get install -y wget +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh curl +RUN apt-get update && apt-get install -y --no-install-recommends wget + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # install audio operator requirements -COPY ./core/operators/audio_vec_embedding_requirements.txt /app/core/operators/audio_vec_embedding_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/audio_vec_embedding_requirements.txt +COPY --chown=python:python ./core/operators/audio_vec_embedding_requirements.txt /usr/app/core/operators/audio_vec_embedding_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/audio_vec_embedding_requirements.txt ### AWS Graviton Optimization ### @@ -47,7 +56,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -COPY . /app +COPY --chown=python:python . /usr/app EXPOSE 7000 -FROM base AS debug \ No newline at end of file +FROM base AS debug +USER 999 diff --git a/src/worker/vidvec/Dockerfile.video_worker b/src/worker/vidvec/Dockerfile.video_worker index 0adfc4ed..7b18178a 100644 --- a/src/worker/vidvec/Dockerfile.video_worker +++ b/src/worker/vidvec/Dockerfile.video_worker @@ -1,21 +1,30 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH -RUN apt-get update && apt-get install -y vim zsh curl +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh curl + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # install video operator requirements -COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt RUN apt-get purge -y --auto-remove \ gcc build-essential \ @@ -23,7 +32,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -COPY . /app +COPY --chown=python:python . /usr/app EXPOSE 7000 -FROM base AS debug \ No newline at end of file +FROM base AS debug +USER 999 diff --git a/src/worker/vidvec/Dockerfile.video_worker.graviton b/src/worker/vidvec/Dockerfile.video_worker.graviton index 64e379e9..f0f2d5be 100644 --- a/src/worker/vidvec/Dockerfile.video_worker.graviton +++ b/src/worker/vidvec/Dockerfile.video_worker.graviton @@ -1,21 +1,30 @@ FROM python:3.11-slim-bullseye AS base + RUN apt-get update \ && apt-get -y upgrade \ && apt-get install -y \ --no-install-recommends gcc build-essential \ --no-install-recommends libgl1-mesa-glx libglib2.0-0 \ --no-install-recommends python3-dev -ENV PATH=/root/.local/bin:$PATH -RUN apt-get update && apt-get install -y vim zsh curl +RUN apt-get update && apt-get install -y --no-install-recommends vim zsh curl + +# Set python user +RUN groupadd -g 999 python && \ + useradd --create-home -r -u 999 -g python python +RUN mkdir /usr/app && chown python:python /usr/app +WORKDIR /usr/app + +# Set venv +RUN python -m venv /usr/app/venv && chown -R python:python /usr/app/venv +ENV PATH="/usr/app/venv/bin:$PATH" RUN pip install --no-cache-dir --upgrade pip -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --user -r requirements.txt +COPY --chown=python:python requirements.txt /usr/app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt # video requirments file -COPY ./core/operators/vid_vec_rep_resnet_requirements.txt /app/core/operators/vid_vec_rep_resnet_requirements.txt -RUN pip install --no-cache-dir --user -r /app/core/operators/vid_vec_rep_resnet_requirements.txt +COPY --chown=python:python ./core/operators/vid_vec_rep_resnet_requirements.txt /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt +RUN pip install --no-cache-dir -r /usr/app/core/operators/vid_vec_rep_resnet_requirements.txt ### AWS Graviton Optimization ### @@ -46,7 +55,8 @@ RUN apt-get purge -y --auto-remove \ python3-dev \ && rm -rf /var/lib/apt/lists/* -COPY . /app +COPY --chown=python:python . /usr/app EXPOSE 7000 -FROM base AS debug \ No newline at end of file +FROM base AS debug +USER 999 From bc2b01c1e24520abb2e0289b7479627f7997bcd3 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 16:58:31 +0530 Subject: [PATCH 2/9] fix (security): Renamed graviton dockerfiles for detection by trivy --- .github/workflows/docker-push-vidvec-benchmark-test.yml | 2 +- .github/workflows/docker-push-vidvec-benchmark.yml | 2 +- .github/workflows/merge-main.yml | 4 ++-- ...dding.graviton => Dockerfile.audio_vec_embedding_graviton} | 0 ...resnet.graviton => Dockerfile.vid_vec_rep_resnet_graviton} | 0 ...audio_worker.graviton => Dockerfile.audio_worker_graviton} | 0 ...video_worker.graviton => Dockerfile.video_worker_graviton} | 0 7 files changed, 4 insertions(+), 4 deletions(-) rename src/benchmark/audiovec/{Dockerfile.audio_vec_embedding.graviton => Dockerfile.audio_vec_embedding_graviton} (100%) rename src/benchmark/vidvec/{Dockerfile.vid_vec_rep_resnet.graviton => Dockerfile.vid_vec_rep_resnet_graviton} (100%) rename src/worker/audiovec/{Dockerfile.audio_worker.graviton => Dockerfile.audio_worker_graviton} (100%) rename src/worker/vidvec/{Dockerfile.video_worker.graviton => Dockerfile.video_worker_graviton} (100%) diff --git a/.github/workflows/docker-push-vidvec-benchmark-test.yml b/.github/workflows/docker-push-vidvec-benchmark-test.yml index 03b23c1e..4fd98045 100644 --- a/.github/workflows/docker-push-vidvec-benchmark-test.yml +++ b/.github/workflows/docker-push-vidvec-benchmark-test.yml @@ -36,7 +36,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ./src/ - file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton + file: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton platforms: linux/arm64 push: true tags: tattletech/feluda-operator-vidvec:benchmark-arm64-latest-test diff --git a/.github/workflows/docker-push-vidvec-benchmark.yml b/.github/workflows/docker-push-vidvec-benchmark.yml index 0ad6c6e4..314e6421 100644 --- a/.github/workflows/docker-push-vidvec-benchmark.yml +++ b/.github/workflows/docker-push-vidvec-benchmark.yml @@ -37,7 +37,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} name: tattletech/feluda-operator-vidvec workdir: src/ - dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton + dockerfile: benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton tags: benchmark-arm64-latest platforms: linux/arm64 diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml index a53cb275..81140ac3 100644 --- a/.github/workflows/merge-main.yml +++ b/.github/workflows/merge-main.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} name: tattletech/feluda-operator-vidvec workdir: src/ - dockerfile: worker/vidvec/Dockerfile.video_worker.graviton + dockerfile: worker/vidvec/Dockerfile.video_worker_graviton tags: worker-arm64-${{ steps.release.outputs.tag }} platforms: linux/arm64 @@ -91,7 +91,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} name: tattletech/feluda-operator-audiovec workdir: src/ - dockerfile: worker/audiovec/Dockerfile.audio_worker.graviton + dockerfile: worker/audiovec/Dockerfile.audio_worker_graviton tags: worker-arm64-${{ steps.release.outputs.tag }} platforms: linux/arm64 diff --git a/src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton b/src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton similarity index 100% rename from src/benchmark/audiovec/Dockerfile.audio_vec_embedding.graviton rename to src/benchmark/audiovec/Dockerfile.audio_vec_embedding_graviton diff --git a/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton b/src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton similarity index 100% rename from src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet.graviton rename to src/benchmark/vidvec/Dockerfile.vid_vec_rep_resnet_graviton diff --git a/src/worker/audiovec/Dockerfile.audio_worker.graviton b/src/worker/audiovec/Dockerfile.audio_worker_graviton similarity index 100% rename from src/worker/audiovec/Dockerfile.audio_worker.graviton rename to src/worker/audiovec/Dockerfile.audio_worker_graviton diff --git a/src/worker/vidvec/Dockerfile.video_worker.graviton b/src/worker/vidvec/Dockerfile.video_worker_graviton similarity index 100% rename from src/worker/vidvec/Dockerfile.video_worker.graviton rename to src/worker/vidvec/Dockerfile.video_worker_graviton From ac487f93c3a8e35f835f39ab1e2bb257978d16a1 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:23:27 +0530 Subject: [PATCH 3/9] ci (security): Added IaC scan with Trivy --- .github/workflows/pr-security.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index 1db8f42b..03bfdb68 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -64,3 +64,19 @@ jobs: local: true inputs: | ./src/core/operators/vid_vec_rep_resnet_requirements.txt + + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'HIGH,CRITICAL' + scanners: 'vuln,misconfig,secret' + skip-dirs: '.vscode,docs' + exit-code: '1' +# - name: Upload Trivy scan results to GitHub Security tab +# uses: github/codeql-action/upload-sarif@v2 +# with: +# sarif_file: 'trivy-results.sarif' From e9afff0b53bbb35e284bfd4ca279e0021910e7a3 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:34:28 +0530 Subject: [PATCH 4/9] ci: Enabled trivy result upload to github codeql --- .github/workflows/pr-security.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index 03bfdb68..f9d8db30 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -76,7 +76,7 @@ jobs: scanners: 'vuln,misconfig,secret' skip-dirs: '.vscode,docs' exit-code: '1' -# - name: Upload Trivy scan results to GitHub Security tab -# uses: github/codeql-action/upload-sarif@v2 -# with: -# sarif_file: 'trivy-results.sarif' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' From 1bcd94b8652610cd59597e3bb372d3a3747ec54f Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:40:43 +0530 Subject: [PATCH 5/9] ci: Added githbu codeql sarif upload permissions --- .github/workflows/pr-security.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index f9d8db30..7bb09ac9 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -14,6 +14,9 @@ on: jobs: checks: if: github.event.pull_request.draft == false + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results name: Run security checks runs-on: ubuntu-latest steps: From 7b1725375901b441ff7ed506d6c76ce2fdbd7ae3 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:44:48 +0530 Subject: [PATCH 6/9] ci: Removed failure condition for trivy scan to allow sarif upload --- .github/workflows/pr-security.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index 7bb09ac9..d86c4799 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -78,7 +78,6 @@ jobs: severity: 'HIGH,CRITICAL' scanners: 'vuln,misconfig,secret' skip-dirs: '.vscode,docs' - exit-code: '1' - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2 with: From 03e48eadcc3aca6d2c302817c641ac2f0e7d5fe8 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:49:18 +0530 Subject: [PATCH 7/9] ci: Updated codeql-action version --- .github/workflows/pr-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index d86c4799..cd20efdd 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -79,6 +79,6 @@ jobs: scanners: 'vuln,misconfig,secret' skip-dirs: '.vscode,docs' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' From b105ac54971bb0e55d2d54b2a82eb1e41d119cd9 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:12:22 +0530 Subject: [PATCH 8/9] ci: Added exit code with limit sarif severities and always upload results --- .github/workflows/pr-security.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index cd20efdd..55e38544 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -75,10 +75,13 @@ jobs: ignore-unfixed: true format: 'sarif' output: 'trivy-results.sarif' + limit-severities-for-sarif: true severity: 'HIGH,CRITICAL' scanners: 'vuln,misconfig,secret' skip-dirs: '.vscode,docs' + exit-code: '1' - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v3 + if: always() with: sarif_file: 'trivy-results.sarif' From ae1ceb4f4fa2e399ec0ba2137bc9cb24e5925189 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:20:32 +0530 Subject: [PATCH 9/9] ci: fixed scanners option --- .github/workflows/pr-security.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-security.yml b/.github/workflows/pr-security.yml index 55e38544..099ebf65 100644 --- a/.github/workflows/pr-security.yml +++ b/.github/workflows/pr-security.yml @@ -77,7 +77,7 @@ jobs: output: 'trivy-results.sarif' limit-severities-for-sarif: true severity: 'HIGH,CRITICAL' - scanners: 'vuln,misconfig,secret' + scanners: 'vuln,config,secret' skip-dirs: '.vscode,docs' exit-code: '1' - name: Upload Trivy scan results to GitHub Security tab