Merge topic/dockerperms to address issue #137 (#138) (#139)

This pull request adds the some new environment variables for Malcolm to address #137

* `PUID` and `PGID`
  * Docker runs all of its containers as the privileged `root` user by default. For better security, Malcolm immediately drops to non-privileged user accounts for executing internal processes wherever possible. The `PUID` (**p**rocess **u**ser **ID**) and `PGID` (**p**rocess **g**roup **ID**) environment variables allow Malcolm to map internal non-privileged user accounts to a corresponding [user account](https://en.wikipedia.org/wiki/User_identifier) on the host.

Additionally, this pull request additionally moves all remaining process that can be run non-privileged to run as non-privileged.

Each docker container now has the following in its Dockerfile (this example is from the zeek container, they're all similar but may have different specific values):

```
ARG DEFAULT_UID=1000
ARG DEFAULT_GID=1000
ENV DEFAULT_UID $DEFAULT_UID
ENV DEFAULT_GID $DEFAULT_GID
ENV PUSER "zeek"
ENV PGROUP "zeek"
ENV PUSER_PRIV_DROP true
```

The entrypoint of each docker comtainer is now [docker-uid-gid-setup.sh](https://github.com/idaholab/Malcolm/blob/master/shared/bin/docker-uid-gid-setup.sh), which does the following:

1. changes the UID and GID of the default (1000:1000) user to match the PUID:PGID provided
2. finds any files *inside* the docker image owned by those IDs and chown them
3. if required, execs the container command by dropping privileges to the unprivileged user

Additionally, control.py (used for start, restart, etc.) will now error out it run as root rather than just running with a bunch of errors. Malcolm should not be run as a root user.

Dockerfiles/curator.Dockerfile

@@ -10,6 +10,17 @@ LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
 LABEL org.opencontainers.image.title='malcolmnetsec/elastalert'
 LABEL org.opencontainers.image.description='Malcolm container providing curation for Elasticsearch indices'
 
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "curator"
+ENV PGROUP "curator"
+ENV PUSER_PRIV_DROP true
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM xterm
+
 ARG ES_HOST=elasticsearch
 ARG ES_PORT=9200
 ARG CURATOR_TIMEOUT=120
@@ -44,39 +55,48 @@ ENV CURATOR_SNAPSHOT_REPO $CURATOR_SNAPSHOT_REPO
 ENV CURATOR_SNAPSHOT_COMPRESSED $CURATOR_SNAPSHOT_COMPRESSED
 ENV CURATOR_SNAPSHOT_DISABLED $CURATOR_SNAPSHOT_DISABLED
 
-ENV DEBIAN_FRONTEND noninteractive
+ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-amd64"
+ENV SUPERCRONIC "supercronic-linux-amd64"
+ENV SUPERCRONIC_SHA1SUM "5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ENV SUPERCRONIC_CRONTAB "/etc/crontab"
+
 ENV CURATOR_VERSION "5.8.1"
 ENV CRON "5 0 * * *"
 ENV CONFIG_FILE "/config/config_file.yml"
 ENV ACTION_FILE "/config/action_file.yml"
-ENV CURATOR_USER "curator"
 
 RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
     apt-get update && \
     apt-get  -y -q install \
       build-essential \
-      cron \
       curl \
       procps \
       psmisc \
       python3 \
       python3-dev \
       python3-pip && \
     pip3 install elasticsearch-curator==${CURATOR_VERSION} && \
-    groupadd --gid 1000 ${CURATOR_USER} && \
-      useradd -M --uid 1000 --gid 1000 ${CURATOR_USER} && \
+    groupadd --gid ${DEFAULT_GID} ${PUSER} && \
+      useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
     apt-get -q -y --purge remove guile-2.2-libs python3-dev build-essential && \
       apt-get -q -y autoremove && \
       apt-get clean && \
       rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    bash -c 'echo -e "${CRON} su -c \"/usr/local/bin/curator --config ${CONFIG_FILE} ${ACTION_FILE}\" ${CURATOR_USER} >/proc/1/fd/1 2>/proc/1/fd/2\n@reboot su -c \"/usr/local/bin/elastic_search_status.sh -w && /usr/local/bin/register-elasticsearch-snapshot-repo.sh\" ${CURATOR_USER} >/proc/1/fd/1 2>/proc/1/fd/2" | crontab -'
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+      echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+      chmod +x "$SUPERCRONIC" && \
+      mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+      ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
+    bash -c 'echo -e "${CRON} /usr/local/bin/curator --config ${CONFIG_FILE} ${ACTION_FILE}" > ${SUPERCRONIC_CRONTAB}'
 
-ADD shared/bin/cron_env_deb.sh /usr/local/bin/
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/elastic_search_status.sh /usr/local/bin/
 ADD curator/scripts /usr/local/bin/
 ADD curator/config /config/
 
-CMD ["/usr/local/bin/cron_env_deb.sh"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+
+CMD ["/usr/local/bin/docker-entrypoint.sh"]
 
 
 # to be populated at build-time:

Dockerfiles/elastalert.Dockerfile

@@ -10,26 +10,35 @@ LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
 LABEL org.opencontainers.image.title='malcolmnetsec/elastalert'
 LABEL org.opencontainers.image.description='Malcolm container providing an alerting framework for Elasticsearch'
 
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "node"
+ENV PGROUP "node"
+ENV PUSER_PRIV_DROP true
+
+ENV TERM xterm
 
 USER root
 
 RUN apk update && \
-    apk add bash curl && \
+    apk add bash curl shadow && \
     rm -rf /var/cache/apk/*
 
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD elastalert/elastalert-start.sh /usr/local/bin/
 ADD shared/bin/elastic_search_status.sh /usr/local/bin/
 
 RUN chmod +x /usr/local/bin/elastalert-start.sh && \
     mkdir -p /opt/elastalert/server_data/tests && \
-    chown -R node:node /opt
+    chown -R ${PUSER}:${PGROUP} /opt
 
 VOLUME ["/opt/elastalert/server_data"]
 
-USER node
-
-ENTRYPOINT ["/usr/local/bin/elastalert-start.sh"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
 
+CMD ["/usr/local/bin/elastalert-start.sh"]
 
 # to be populated at build-time:
 ARG BUILD_DATE

Dockerfiles/elasticsearch.Dockerfile

@@ -0,0 +1,35 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:7.6.2
+
+# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+LABEL maintainer="[email protected]"
+LABEL org.opencontainers.image.authors='[email protected]'
+LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm'
+LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/master/README.md'
+LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm'
+LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
+LABEL org.opencontainers.image.title='malcolmnetsec/elasticsearch-oss'
+LABEL org.opencontainers.image.description='Malcolm container providing Elasticsearch (the Apache-licensed variant)'
+
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "elasticsearch"
+ENV PGROUP "elasticsearch"
+ENV PUSER_PRIV_DROP true
+
+ENV TERM xterm
+
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
+
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker-entrypoint.sh"]
+
+
+# to be populated at build-time:
+ARG BUILD_DATE
+ARG MALCOLM_VERSION
+ARG VCS_REVISION
+
+LABEL org.opencontainers.image.created=$BUILD_DATE
+LABEL org.opencontainers.image.version=$MALCOLM_VERSION
+LABEL org.opencontainers.image.revision=$VCS_REVISION

Dockerfiles/file-monitor.Dockerfile

@@ -10,8 +10,16 @@ LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
 LABEL org.opencontainers.image.title='malcolmnetsec/file-monitor'
 LABEL org.opencontainers.image.description='Malcolm container for scanning files extracted by Zeek'
 
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "monitor"
+ENV PGROUP "monitor"
+ENV PUSER_PRIV_DROP true
 
 ENV DEBIAN_FRONTEND noninteractive
+ENV TERM xterm
 
 ARG ZEEK_EXTRACTOR_PATH=/data/zeek/extract_files
 ARG ZEEK_LOG_DIRECTORY=/data/zeek/logs
@@ -74,31 +82,32 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       python3-requests \
       python3-zmq && \
     pip3 install clamd supervisor && \
-    mkdir -p /var/log/supervisor && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove python3-dev build-essential && \
       apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
       apt-get clean && \
       rm -rf /var/lib/apt/lists/* && \
     wget -O /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \
       wget -O /var/lib/clamav/daily.cvd http://database.clamav.net/daily.cvd && \
       wget -O /var/lib/clamav/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
-    groupadd --gid 1000 monitor && \
-      useradd -M --uid 1000 --gid 1000 monitor && \
+    groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
+      useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
+      usermod -a -G tty ${PUSER} && \
     mkdir -p /var/log/clamav /var/lib/clamav && \
-      chown -R monitor:monitor /var/log/clamav  /var/lib/clamav && \
+      chown -R ${PUSER}:${PGROUP} /var/log/clamav  /var/lib/clamav && \
       chmod -R 750 /var/log/clamav  /var/lib/clamav && \
     sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/clamd.conf && \
-      sed -i 's/^User .*$/User monitor/g' /etc/clamav/clamd.conf && \
+      sed -i "s/^User .*$/User ${PUSER}/g" /etc/clamav/clamd.conf && \
       sed -i "s|^LocalSocket .*$|LocalSocket $CLAMD_SOCKET_FILE|g" /etc/clamav/clamd.conf && \
-      sed -i 's/^LocalSocketGroup .*$/LocalSocketGroup monitor/g' /etc/clamav/clamd.conf && \
+      sed -i "s/^LocalSocketGroup .*$/LocalSocketGroup ${PGROUP}/g" /etc/clamav/clamd.conf && \
       sed -i "s/^MaxFileSize .*$/MaxFileSize $EXTRACTED_FILE_MAX_BYTES/g" /etc/clamav/clamd.conf && \
       sed -i "s/^MaxScanSize .*$/MaxScanSize $(echo "$EXTRACTED_FILE_MAX_BYTES * 4" | bc)/g" /etc/clamav/clamd.conf && \
       echo "TCPSocket 3310" >> /etc/clamav/clamd.conf && \
     if ! [ -z $HTTPProxyServer ]; then echo "HTTPProxyServer $HTTPProxyServer" >> /etc/clamav/freshclam.conf; fi && \
       if ! [ -z $HTTPProxyPort   ]; then echo "HTTPProxyPort $HTTPProxyPort" >> /etc/clamav/freshclam.conf; fi && \
       sed -i 's/^Foreground .*$/Foreground true/g' /etc/clamav/freshclam.conf && \
-      sed -i 's/^DatabaseOwner .*$/DatabaseOwner monitor/g' /etc/clamav/freshclam.conf
+      sed -i "s/^DatabaseOwner .*$/DatabaseOwner ${PUSER}/g" /etc/clamav/freshclam.conf
 
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/zeek_carve_*.py /usr/local/bin/
 ADD shared/bin/malass_client.py /usr/local/bin/
 ADD file-monitor/supervisord.conf /etc/supervisord.conf
@@ -109,7 +118,9 @@ VOLUME ["/var/lib/clamav"]
 
 EXPOSE 3310
 
-CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+
+CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
 
 
 # to be populated at build-time:

Dockerfiles/file-upload.Dockerfile

@@ -1,15 +1,8 @@
 FROM debian:buster-slim AS build
 
 # Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
-LABEL maintainer="[email protected]"
-LABEL org.opencontainers.image.authors='[email protected]'
-LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm'
-LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/master/README.md'
-LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm'
-LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
-LABEL org.opencontainers.image.title='malcolmnetsec/file-upload'
-LABEL org.opencontainers.image.description='Malcolm container providing an interface for uploading PCAP files and Zeek logs for processing'
 
+ENV DEBIAN_FRONTEND noninteractive
 
 ARG SITE_NAME="Capture File and Log Archive Upload"
 
@@ -32,11 +25,31 @@ RUN apt-get update && \
 
 FROM debian:buster-slim AS runtime
 
-COPY --from=build /jQuery-File-Upload/ /var/www/upload/
+LABEL maintainer="[email protected]"
+LABEL org.opencontainers.image.authors='[email protected]'
+LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm'
+LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/master/README.md'
+LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm'
+LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
+LABEL org.opencontainers.image.title='malcolmnetsec/file-upload'
+LABEL org.opencontainers.image.description='Malcolm container providing an interface for uploading PCAP files and Zeek logs for processing'
+
+ARG DEFAULT_UID=33
+ARG DEFAULT_GID=33
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "www-data"
+ENV PGROUP "www-data"
+# not dropping privileges globally in this container as required to run SFTP server. this can
+# be handled by supervisord instead on an as-needed basis, and/or php-fpm/nginx itself
+# will drop privileges to www-data as well.
+ENV PUSER_PRIV_DROP false
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
+COPY --from=build /jQuery-File-Upload/ /var/www/upload/
+
 RUN apt-get update && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends \
       wget \
@@ -52,6 +65,7 @@ RUN apt-get update && \
     apt-get clean -y -q && \
     rm -rf /var/lib/apt/lists/*
 
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD docs/images/logo/Malcolm_banner.png /var/www/upload/Malcolm_banner.png
 ADD file-upload/docker-entrypoint.sh /docker-entrypoint.sh
 ADD file-upload/jquery-file-upload/bootstrap.min.css /var/www/upload/bower_components/bootstrap/dist/css/bootstrap.min.css
@@ -73,14 +87,15 @@ RUN mkdir -p /var/run/sshd /var/www/upload/server/php/chroot /run/php && \
   chmod 775 /var/www/upload/server/php/chroot/files && \
   chmod 755 /var /var/www /var/www/upload /var/www/upload/server /var/www/upload/server/php \
             /var/www/upload/server/php/chroot && \
-  echo "Put your files into /files. Don't use subdirectories.\nThey cannot be accessed via the web user interface!" \
+  echo "Put your files into /files. Don't use subdirectories." \
       >/var/www/upload/server/php/chroot/README.txt && \
   rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /var/tmp/* /var/www/upload/server/php/chroot/files/.gitignore /tmp/sshd_config
 
 VOLUME [ "/var/www/upload/server/php/chroot/files" ]
 EXPOSE 22 80
 
-ENTRYPOINT [ "/docker-entrypoint.sh" ]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"]
+
 CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf", "-u", "root", "-n"]
 
 

Dockerfiles/filebeat.Dockerfile

@@ -10,6 +10,19 @@ LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
 LABEL org.opencontainers.image.title='malcolmnetsec/filebeat-oss'
 LABEL org.opencontainers.image.description='Malcolm container providing Filebeat (the Apache-licensed variant)'
 
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "filebeat"
+ENV PGROUP "filebeat"
+# not dropping privileges globally: supervisord will take care of it
+# on a case-by-case basis so that one script (filebeat-watch-zeeklogs-uploads-folder.sh)
+# can chown uploaded files
+ENV PUSER_PRIV_DROP false
+
+ENV TERM xterm
+
 ARG FILEBEAT_LOG_CLEANUP_MINUTES=0
 ARG FILEBEAT_ZIP_CLEANUP_MINUTES=0
 ARG FILEBEAT_SCAN_FREQUENCY=10s
@@ -25,29 +38,39 @@ ARG FILEBEAT_NGINX_LOG_PATH="/data/nginx"
 ARG NGINX_LOG_ACCESS_AND_ERRORS=false
 ARG AUTO_TAG=true
 
+ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-amd64"
+ENV SUPERCRONIC "supercronic-linux-amd64"
+ENV SUPERCRONIC_SHA1SUM "5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ENV SUPERCRONIC_CRONTAB "/etc/crontab"
+
 USER root
 
 RUN yum install -y epel-release && \
-    yum update -y && \
-    yum install -y cronie inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python-setuptools python-pip && \
-    yum clean all && \
+        yum update -y && \
+        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz p7zip p7zip-plugins unar python-setuptools python-pip && \
+        yum clean all && \
+    ln -sr /usr/sbin/fuser /bin/fuser && \
     easy_install supervisor && \
     pip install patool entrypoint2 pyunpack python-magic ordered-set==3.1.1 && \
-    ln -sr /usr/sbin/fuser /bin/fuser
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+      echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+      chmod +x "$SUPERCRONIC" && \
+      mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+      ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic
 
-ADD shared/bin/cron_env_centos.sh /data/
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
 ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
 ADD filebeat/scripts /data/
 ADD shared/bin/elastic_search_status.sh /data/
 ADD filebeat/supervisord.conf /etc/supervisord.conf
-RUN mkdir -p /var/log/supervisor /usr/share/filebeat-nginx/data && \
-    chown -R root:filebeat /usr/share/filebeat-nginx && \
+RUN mkdir -p /usr/share/filebeat-nginx/data && \
+    chown -R root:${PGROUP} /usr/share/filebeat-nginx && \
     cp -a /usr/share/filebeat/module /usr/share/filebeat-nginx/module && \
     chmod 750 /usr/share/filebeat-nginx && \
     chmod 770 /usr/share/filebeat-nginx/data && \
     chmod 755 /data/*.sh /data/*.py && \
-    (echo -e "* * * * * su -c /data/filebeat-process-zeek-folder.sh filebeat >/dev/null 2>&1\n*/5 * * * * su -c /data/filebeat-clean-zeeklogs-processed-folder.py filebeat >/dev/null 2>&1" | crontab -)
+    (echo -e "* * * * * /data/filebeat-process-zeek-folder.sh\n*/5 * * * * /data/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
 
 ENV FILEBEAT_LOG_CLEANUP_MINUTES $FILEBEAT_LOG_CLEANUP_MINUTES
 ENV FILEBEAT_ZIP_CLEANUP_MINUTES $FILEBEAT_ZIP_CLEANUP_MINUTES
@@ -70,6 +93,8 @@ ENV PATH="/data:${PATH}"
 
 VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data"]
 
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-u", "root", "-n"]