Move mwdb.cert.pl deployment to separate repository

Author: psrok1

.dockerignore

@@ -0,0 +1,9 @@
+*.pyc
+venv/
+.vscode
+.idea
+.mypy_cache
+uploads/
+malwarecage.ini
+malwarecage.example.ini
+

.gitignore

@@ -13,4 +13,8 @@ mwdb-vars.env
 postgres-vars.env
 .mypy_cache/
 /malwarefront/extensions/plugins/
-/malwarefront/extensions/package.json
+/malwarefront/extensions/package.json
+# These directories are user-controlled
+/plugins
+/mail_templates
+

.gitlab-ci.yml

@@ -3,6 +3,7 @@ image: certpl/docker-ci-base:latest
 variables:
   DOCKER_REGISTRY: dr.cert.pl/malwarecage
   GIT_SUBMODULE_STRATEGY: none
+  DOCKER_BUILDKIT: 1
 
 services:
   - docker:dind
@@ -28,7 +29,7 @@ before_script:
   - docker login -u "$DOCKER_REGISTRY_LOGIN" -p "$DOCKER_REGISTRY_PASSWORD" "$DOCKER_REGISTRY"
   - mkdir -p artifacts/test
   - export CI_COMMIT_REF_NAME=`echo -n $CI_COMMIT_REF_NAME | sed 's#/#-#g'`
-  
+
 build_mwdb:
   stage: build
   script:
@@ -42,8 +43,7 @@ build_mwdb_tests:
 build_malwarefront:
   stage: build
   script:
-    - ./build-image.sh "$DOCKER_REGISTRY/malwarecage_malwarefront_build" "--target build -f ./deploy/docker/Dockerfile-malwarefront ."
-    - ./build-image.sh "$DOCKER_REGISTRY/malwarecage_malwarefront" "--cache-from \"$DOCKER_REGISTRY/malwarecage_malwarefront_build:$CI_COMMIT_REF_NAME\" --cache-from \"$DOCKER_REGISTRY/malwarecage_malwarefront_build:latest\" -f ./deploy/docker/Dockerfile-malwarefront ."
+    - ./build-image.sh "$DOCKER_REGISTRY/malwarecage_malwarefront" ". -f ./deploy/docker/Dockerfile-malwarefront"
 
 test_mwdb:
   stage: test
@@ -72,34 +72,3 @@ test_mwdb:
       - artifacts/test/malwarefront.log
     when: always
     expire_in: 1 week
-
-deploy_staging:
-  stage: deploy
-  only:
-    - master
-  environment:
-    name: staging
-    url: https://mwdb-st.cert.pl
-  script:
-    - kubectl config set-cluster cert --server="https://kapi.cert.pl" --insecure-skip-tls-verify=true
-    - kubectl config set-credentials gitlab-ci --token "$KUBE_TOKEN"
-    - kubectl config set-context default --cluster=cert --user=gitlab-ci
-    - kubectl config use-context default
-    - kubectl -n malwarecage-st set image deployment.v1.apps/mwdb mwdb-container=$DOCKER_REGISTRY/malwarecage_mwdb:$CI_COMMIT_SHA
-    - kubectl -n malwarecage-st set image deployment.v1.apps/malwarefront malwarefront-container=$DOCKER_REGISTRY/malwarecage_malwarefront:$CI_COMMIT_SHA
-
-deploy:
-  stage: deploy
-  when: manual
-  only:
-    - master
-  environment:
-    name: production
-    url: https://mwdb.cert.pl
-  script:
-    - kubectl config set-cluster cert --server="https://kapi.cert.pl" --insecure-skip-tls-verify=true
-    - kubectl config set-credentials gitlab-ci --token "$KUBE_TOKEN"
-    - kubectl config set-context default --cluster=cert --user=gitlab-ci
-    - kubectl config use-context default
-    - kubectl -n malwarecage-prod set image deployment.v1.apps/mwdb mwdb-container=$DOCKER_REGISTRY/malwarecage_mwdb:$CI_COMMIT_SHA
-    - kubectl -n malwarecage-prod set image deployment.v1.apps/malwarefront malwarefront-container=$DOCKER_REGISTRY/malwarecage_malwarefront:$CI_COMMIT_SHA

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "plugins"]
-	path = plugins
-	url = [email protected]:mlwr/malwarecage-plugins.git

build-image.sh

@@ -1,22 +1,10 @@
 #!/bin/sh
 
-echo "Pulling $1:$CI_COMMIT_REF_NAME..."
-time docker pull "$1:$CI_COMMIT_REF_NAME" > /dev/null
-
-if test $? -ne 0
-then
-    echo "Pulling $1:latest..."
-    time docker pull "$1:latest" > /dev/null
-    
-    if test $? -ne 0
-    then
-        echo "No cached images present, will build everything from scratch..."
-    fi
-fi
-
-docker images
-
-time docker build --cache-from "$1:$CI_COMMIT_REF_NAME" --cache-from "$1:latest" -t "$1:$CI_COMMIT_SHA" -t "$1:$CI_COMMIT_REF_NAME" $2 || exit 1
+docker build --cache-from "$1:$CI_COMMIT_REF_NAME" \
+             --cache-from "$1:latest" \
+             --tag "$1:$CI_COMMIT_SHA" \
+             --tag "$1:$CI_COMMIT_REF_NAME" \
+             --build-arg BUILDKIT_INLINE_CACHE=1 $2 || exit 1
 
 echo "Pushing as $1:$CI_COMMIT_SHA"
 time docker push "$1:$CI_COMMIT_SHA" > /dev/null

deploy/docker/Dockerfile

@@ -2,10 +2,10 @@ FROM tiangolo/uwsgi-nginx-flask:python3.6-alpine3.8
 
 LABEL maintainer="[email protected]"
 
+RUN apk add --no-cache postgresql-client postgresql-dev libmagic
+
 COPY requirements.txt /tmp/requirements.txt
-COPY plugins /tmp/plugins/
-RUN apk add --no-cache postgresql-client postgresql-dev libmagic \
-    && apk add --no-cache -t build libffi libffi-dev py3-cffi build-base python3-dev automake m4 perl autoconf libtool \
+RUN apk add --no-cache -t build libffi libffi-dev py3-cffi build-base python3-dev automake m4 perl autoconf libtool \
     && wget -O /tmp/ssdeep.tar.gz https://github.com/ssdeep-project/ssdeep/releases/download/release-2.14.1/ssdeep-2.14.1.tar.gz \
     && cd /tmp \
     && tar -xopf /tmp/ssdeep.tar.gz \
@@ -14,19 +14,35 @@ RUN apk add --no-cache postgresql-client postgresql-dev libmagic \
     && make \
     && make install \
     && cd /tmp && pip --no-cache-dir install -r requirements.txt \
-    && ls plugins/*/requirements.txt | xargs -i,, pip --no-cache-dir install -r ,, \
     && apk del build
 
-COPY . /app
+# Install plugin requirements 
+# Because of Docker limitations: at least one file must exist, so original requirements.txt is appended
 
-# Create a /app/uploads directory
-RUN mkdir -p /app/uploads/
+COPY requirements.txt plugins/requirements-*.txt /tmp/
 
-# Give +r to everything in /app and +x for directories
-RUN chmod u=rX,go= -R /app
+RUN ls /tmp/requirements-*.txt | xargs -i,, pip --no-cache-dir install -r ,,
 
-# Give rwx permissions to /app/uploads for the current user
-RUN chmod 700 /app/uploads/
+# Copy backend files
+
+COPY prestart.sh uwsgi.ini plugin_engine.py app.py version.py /app/
+
+COPY core /app/core
+COPY migrations /app/migrations
+COPY model /app/model
+COPY resources /app/resources
 
+# Copy user-controlled plugins
+COPY plugins /app/plugins
+
+# Copy user-controlled mail templates
+COPY mail_templates /app/mail_templates
+
+# Create a /app/uploads directory
+# Give +r to everything in /app and +x for directories
+# Give rwx permissions to /app/uploads for the current user
 # By default everything is owned by root - change owner to nobody
-RUN chown nobody:nobody -R /app
+RUN mkdir -p /app/uploads/ && \
+    chmod o=rX -R /app && \
+    chmod 700 /app/uploads/ && \
+    chown nobody:nobody /app/uploads/