Work in progress for zeek and suricata live capture (squashed commit)…

… see #109 commit 30ab5eb Author: SG <[email protected]> Date: Wed Jul 27 11:40:48 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit dda60d7 Author: SG <[email protected]> Date: Wed Jul 27 11:40:30 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit b08e3e7 Author: SG <[email protected]> Date: Wed Jul 27 10:39:08 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 51c8e04 Author: Seth Grover <[email protected]> Date: Wed Jul 27 08:26:01 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 35da23a Author: Seth Grover <[email protected]> Date: Wed Jul 27 07:00:47 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit b6c99f9 Author: Seth Grover <[email protected]> Date: Tue Jul 26 17:17:19 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit ce9149e Author: Seth Grover <[email protected]> Date: Tue Jul 26 16:30:22 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 3aec8b5 Author: Seth Grover <[email protected]> Date: Tue Jul 26 16:08:37 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit a9fe5b2 Author: Seth Grover <[email protected]> Date: Tue Jul 26 15:49:58 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 78f0866 Author: Seth Grover <[email protected]> Date: Tue Jul 26 15:33:31 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 0e4f043 Author: Seth Grover <[email protected]> Date: Tue Jul 26 14:48:56 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 6a16eee Author: Seth Grover <[email protected]> Date: Tue Jul 26 14:05:56 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit ad394b0 Author: Seth Grover <[email protected]> Date: Tue Jul 26 13:53:39 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit c80b465 Author: Seth Grover <[email protected]> Date: Tue Jul 26 13:07:54 2022 -0600 work in progress for zeek- and suricata- live capture (#109) commit 32d5ca1 Author: Seth Grover <[email protected]> Date: Tue Jul 26 12:02:17 2022 -0600 work in progress for zeek- and suricata- live capture (see #109) Signed-off-by: Seth Grover <[email protected]> commit 326db59 Author: Seth Grover <[email protected]> Date: Tue Jul 26 09:53:43 2022 -0600 bump to v6.2.0 for #109
idaholab · Jul 27, 2022 · 8774a1e · 8774a1e
1 parent fe3ae25
commit 8774a1e
Show file tree

Hide file tree

Showing 42 changed files with 512 additions and 214 deletions.
diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml
@@ -93,7 +93,7 @@ jobs:
       -
         name: Build image
         run: |
-          IMAGES=( $(grep image: docker-compose.yml | awk '{print $2}') )
+          IMAGES=( $(grep image: docker-compose.yml | awk '{print $2}' | sort -u) )
           for IMAGE in "${IMAGES[@]}"; do
             REPO_IMAGE="$(echo "$IMAGE" | sed "s@^\(malcolmnetsec\)@ghcr.io/${{ github.repository_owner }}/\1@" | sed "s/:.*/:${{ steps.extract_branch.outputs.branch }}/")"
             docker pull "$REPO_IMAGE" && \

diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
@@ -21,8 +21,8 @@ ENV PUSER_PRIV_DROP true
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
-ARG ZEEK_EXTRACTOR_PATH=/data/zeek/extract_files
-ARG ZEEK_LOG_DIRECTORY=/data/zeek/logs
+ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
+ARG ZEEK_LOG_DIRECTORY=/zeek/logs
 ARG EXTRACTED_FILE_IGNORE_EXISTING=false
 ARG EXTRACTED_FILE_PRESERVATION=quarantined
 ARG EXTRACTED_FILE_WATCHER_START_SLEEP=30
@@ -212,7 +212,7 @@ RUN /usr/bin/freshclam freshclam --config-file=/etc/clamav/freshclam.conf
 
 USER root
 
-WORKDIR /data/zeek/extract_files
+WORKDIR /zeek/extract_files
 
 ENV PATH "${CAPA_DIR}:${PATH}"
 

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
@@ -32,9 +32,10 @@ ARG FILEBEAT_CLOSE_RENAMED=true
 ARG FILEBEAT_CLOSE_REMOVED=true
 ARG FILEBEAT_CLOSE_EOF=true
 ARG FILEBEAT_CLEAN_REMOVED=true
-ARG FILEBEAT_ZEEK_LOG_PATH="/data/zeek/current"
-ARG FILEBEAT_SURICATA_LOG_PATH="/data/suricata"
-ARG FILEBEAT_NGINX_LOG_PATH="/data/nginx"
+ARG FILEBEAT_ZEEK_LOG_PATH="/zeek/current"
+ARG FILEBEAT_ZEEK_LOG_LIVE_PATH="/zeek/live"
+ARG FILEBEAT_SURICATA_LOG_PATH="/suricata"
+ARG FILEBEAT_NGINX_LOG_PATH="/nginx"
 ARG LOG_CLEANUP_MINUTES=0
 ARG ZIP_CLEANUP_MINUTES=0
 ARG NGINX_LOG_ACCESS_AND_ERRORS=false
@@ -77,8 +78,8 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
 ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
 ADD filebeat/filebeat-tcp.yml /usr/share/filebeat-tcp/filebeat-tcp.yml
-ADD filebeat/scripts /data/
-ADD shared/bin/opensearch_status.sh /data/
+ADD filebeat/scripts /usr/local/bin/
+ADD shared/bin/opensearch_status.sh /usr/local/bin/
 ADD filebeat/supervisord.conf /etc/supervisord.conf
 RUN for INPUT in nginx tcp; do \
       mkdir -p /usr/share/filebeat-$INPUT/data; \
@@ -87,8 +88,8 @@ RUN for INPUT in nginx tcp; do \
       chmod 750 /usr/share/filebeat-$INPUT; \
       chmod 770 /usr/share/filebeat-$INPUT/data; \
     done; \
-    chmod 755 /data/*.sh /data/*.py && \
-    (echo -e "* * * * * /data/filebeat-process-zeek-folder.sh\n*/5 * * * * /data/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
+    chmod 755 /usr/local/bin/*.sh /usr/local/bin/*.py && \
+    (echo -e "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
 
 ENV AUTO_TAG $AUTO_TAG
 ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES
@@ -102,6 +103,7 @@ ENV FILEBEAT_CLOSE_REMOVED $FILEBEAT_CLOSE_REMOVED
 ENV FILEBEAT_CLOSE_EOF $FILEBEAT_CLOSE_EOF
 ENV FILEBEAT_CLEAN_REMOVED $FILEBEAT_CLEAN_REMOVED
 ENV FILEBEAT_ZEEK_LOG_PATH $FILEBEAT_ZEEK_LOG_PATH
+ENV FILEBEAT_ZEEK_LOG_LIVE_PATH $FILEBEAT_ZEEK_LOG_LIVE_PATH
 ENV FILEBEAT_SURICATA_LOG_PATH $FILEBEAT_SURICATA_LOG_PATH
 ENV FILEBEAT_NGINX_LOG_PATH $FILEBEAT_NGINX_LOG_PATH
 ENV NGINX_LOG_ACCESS_AND_ERRORS $NGINX_LOG_ACCESS_AND_ERRORS
@@ -120,8 +122,7 @@ ENV FILEBEAT_TCP_PARSE_TARGET_FIELD $FILEBEAT_TCP_PARSE_TARGET_FIELD
 ENV FILEBEAT_TCP_PARSE_DROP_FIELD $FILEBEAT_TCP_PARSE_DROP_FIELD
 ENV FILEBEAT_TCP_TAG $FILEBEAT_TCP_TAG
 ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json"
-ENV FILEBEAT_ZEEK_DIR "/data/zeek/"
-ENV PATH="/data:${PATH}"
+ENV FILEBEAT_ZEEK_DIR "/zeek/"
 
 VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-tcp/data"]
 

diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile
@@ -30,7 +30,7 @@ ENV TERM xterm
 ARG PCAP_ENABLE_TCPDUMP=false
 ARG PCAP_ENABLE_NETSNIFF=false
 # PCAP_IFACE=comma-separated list of capture interfaces
-ARG PCAP_IFACE=eth0
+ARG PCAP_IFACE=lo
 ARG PCAP_NETSNIFF_MAGIC=0xa1b2c3d4
 ARG PCAP_TCPDUMP_FILENAME_PATTERN=%Y%m%d%H%M%S.pcap
 ARG PCAP_ROTATE_MINUTES=30

diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile
@@ -101,7 +101,13 @@ ENV DEFAULT_UID $DEFAULT_UID
 ENV DEFAULT_GID $DEFAULT_GID
 ENV PUSER "suricata"
 ENV PGROUP "suricata"
-ENV PUSER_PRIV_DROP true
+# not dropping privileges globally: supervisord will take care of it
+# for all processes, but first we need root to sure capabilities for
+# traffic capturing tools are in-place before they are started.
+# despite doing setcap here in the Dockerfile, the chown in
+# docker-uid-gid-setup.sh will cause them to be lost, so we need
+# a final check in docker_entrypoint.sh before startup
+ENV PUSER_PRIV_DROP false
 
 ENV SUPERCRONIC_VERSION "0.2.1"
 ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
@@ -125,12 +131,16 @@ COPY --from=builder /suricatabld.tar.gz /suricatabld.tar.gz
 RUN apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get install -q -y --no-install-recommends \
+        bc \
         curl \
+        ethtool \
         file \
         inotify-tools \
+        iproute2 \
         jq \
         less \
         libcap-ng0 \
+        libcap2-bin \
         libevent-2.1-7 \
         libevent-pthreads-2.1-7 \
         libgeoip1 \
@@ -175,6 +185,9 @@ RUN apt-get -q update && \
     ln -sfr /opt/pcap_processor.py /opt/pcap_suricata_processor.py && \
         (echo "*/5 * * * * /opt/eve-clean-logs.sh" > ${SUPERCRONIC_CRONTAB}) && \
     tar xf /suricatabld.tar.gz --strip-components=1 -C / && \
+    chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \
+      setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \
+      setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \
     mkdir -p "$SURICATA_CUSTOM_RULES_DIR" && \
         chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" && \
     apt-get clean && \
@@ -193,19 +206,32 @@ ARG PCAP_PIPELINE_DEBUG=false
 ARG PCAP_PIPELINE_DEBUG_EXTRA=false
 ARG PCAP_MONITOR_HOST=pcap-monitor
 ARG AUTO_TAG=true
+ARG SURICATA_PCAP_PROCESSOR=true
+ARG SURICATA_CRON=true
 ARG SURICATA_AUTO_ANALYZE_PCAP_FILES=false
 ARG SURICATA_CUSTOM_RULES_ONLY=false
 ARG SURICATA_AUTO_ANALYZE_PCAP_THREADS=1
 ARG LOG_CLEANUP_MINUTES=30
+ARG SURICATA_LIVE_CAPTURE=false
+ARG SURICATA_ROTATED_PCAP=false
+# PCAP_IFACE=comma-separated list of capture interfaces
+ARG PCAP_IFACE=lo
+ARG PCAP_FILTER=
 
 ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
-ENV AUTO_TAG $AUTO_TAG
 ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
 ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
+ENV AUTO_TAG $AUTO_TAG
+ENV SURICATA_PCAP_PROCESSOR $SURICATA_PCAP_PROCESSOR
+ENV SURICATA_CRON $SURICATA_CRON
 ENV SURICATA_AUTO_ANALYZE_PCAP_FILES $SURICATA_AUTO_ANALYZE_PCAP_FILES
 ENV SURICATA_AUTO_ANALYZE_PCAP_THREADS $SURICATA_AUTO_ANALYZE_PCAP_THREADS
 ENV SURICATA_CUSTOM_RULES_ONLY $SURICATA_CUSTOM_RULES_ONLY
 ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES
+ENV SURICATA_LIVE_CAPTURE $SURICATA_LIVE_CAPTURE
+ENV SURICATA_ROTATED_PCAP $SURICATA_ROTATED_PCAP
+ENV PCAP_IFACE $PCAP_IFACE
+ENV PCAP_FILTER $PCAP_FILTER
 
 ENV PUSER_CHOWN "$SURICATA_CONFIG_DIR;$SURICATA_MANAGED_DIR;$SURICATA_LOG_DIR;$SURICATA_RUN_DIR"
 

diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
@@ -21,7 +21,13 @@ ENV DEFAULT_UID $DEFAULT_UID
 ENV DEFAULT_GID $DEFAULT_GID
 ENV PUSER "zeeker"
 ENV PGROUP "zeeker"
-ENV PUSER_PRIV_DROP true
+# not dropping privileges globally: supervisord will take care of it
+# for all processes, but first we need root to sure capabilities for
+# traffic capturing tools are in-place before they are started.
+# despite doing setcap here in the Dockerfile, the chown in
+# docker-uid-gid-setup.sh will cause them to be lost, so we need
+# a final check in docker_entrypoint.sh before startup
+ENV PUSER_PRIV_DROP false
 
 # for download and install
 ARG ZEEK_LTS=
@@ -48,20 +54,24 @@ ENV PATH "${ZEEK_DIR}/bin:${PATH}"
 ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/
 
 # build and install system packages, zeek, spicy and plugins
-RUN apt-get -q update && \
+RUN export DEBARCH=$(dpkg --print-architecture) && \
+    apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get install -q -y --no-install-recommends \
+      bc \
       bison \
       ca-certificates \
       ccache \
       cmake \
       curl \
+      ethtool \
       file \
       flex \
       g++ \
       gcc \
       git \
       gnupg2 \
+      iproute2 \
       jq \
       less \
       libatomic1 \
@@ -77,6 +87,7 @@ RUN apt-get -q update && \
       libtcmalloc-minimal4 \
       libunwind8 \
       libzmq5 \
+      linux-headers-$DEBARCH \
       locales-all \
       make \
       moreutils \
@@ -145,7 +156,9 @@ ADD shared/pcaps /tmp/pcaps
 ADD zeek/supervisord.conf /etc/supervisord.conf
 ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/
 ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/
-ADD shared/bin/zeek_intel_setup.sh /usr/local/bin/entrypoint.sh
+ADD zeek/scripts/docker_entrypoint.sh /usr/local/bin/
+ADD shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/
+ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/
 
 # sanity checks to make sure the plugins installed and copied over correctly
 # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
@@ -166,12 +179,20 @@ RUN mkdir -p /tmp/logs && \
 RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \
     useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \
     usermod -a -G tty ${PUSER} && \
+    chown root:${PGROUP} /sbin/ethtool "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \
+      setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \
+      setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek && \
+      setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats && \
     touch "${SUPERCRONIC_CRONTAB}" && \
     chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel "${SUPERCRONIC_CRONTAB}" && \
     ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_zeek_processor.py
 
 #Whether or not to auto-tag logs based on filename
 ARG AUTO_TAG=true
+#Whether or not to start up the pcap_processor script to monitor pcaps
+ARG ZEEK_PCAP_PROCESSOR=true
+#Whether or not to start up supercronic for updating intel definitions
+ARG ZEEK_CRON=true
 #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file
 ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false
 ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
@@ -184,8 +205,15 @@ ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
 ARG PCAP_PIPELINE_DEBUG=false
 ARG PCAP_PIPELINE_DEBUG_EXTRA=false
 ARG PCAP_MONITOR_HOST=pcap-monitor
+ARG ZEEK_LIVE_CAPTURE=false
+ARG ZEEK_ROTATED_PCAP=false
+# PCAP_IFACE=comma-separated list of capture interfaces
+ARG PCAP_IFACE=lo
+ARG PCAP_FILTER=
 
 ENV AUTO_TAG $AUTO_TAG
+ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR
+ENV ZEEK_CRON $ZEEK_CRON
 ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES
 ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS
 ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION
@@ -197,6 +225,10 @@ ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
 ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
 ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA
 ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
+ENV ZEEK_LIVE_CAPTURE $ZEEK_LIVE_CAPTURE
+ENV ZEEK_ROTATED_PCAP $ZEEK_ROTATED_PCAP
+ENV PCAP_IFACE $PCAP_IFACE
+ENV PCAP_FILTER $PCAP_FILTER
 
 # environment variables for zeek runtime tweaks (used in local.zeek)
 ARG ZEEK_DISABLE_HASH_ALL_FILES=
@@ -234,9 +266,11 @@ ENV ZEEK_DISABLE_SPICY_TAILSCALE $ZEEK_DISABLE_SPICY_TAILSCALE
 ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP
 ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD
 
+ENV PUSER_CHOWN "$ZEEK_DIR"
+
 VOLUME ["${ZEEK_DIR}/share/zeek/site/intel"]
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker_entrypoint.sh"]
 
 CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]