diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 0dbedcca7..407f7fc27 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -86,7 +86,7 @@ ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base" ENV YARA_RULES_DIR "/yara-rules" ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base" -ENV CAPA_VERSION "2.0.0" +ENV CAPA_VERSION "3.0.2" ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip" ENV CAPA_DIR "/opt/capa" ENV CAPA_BIN "${CAPA_DIR}/capa" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index e36a9e500..bce08aff1 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -25,7 +25,7 @@ ENV PUSER_PRIV_DROP true # for download and install ARG ZEEK_LTS=1 -ARG ZEEK_VERSION=4.0.3-0 +ARG ZEEK_VERSION=4.0.4-0 ARG SPICY_VERSION=1.2.1 ENV ZEEK_LTS $ZEEK_LTS @@ -152,9 +152,9 @@ ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/ # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh # todo: Bro::LDAP is broken right now, disabled ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 19 -ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(_Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OPENVPN|ANALYZER_SPICY_IPSEC|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP_IP|ICSNPP::BSAP_SERIAL|ICSNPP::ENIP|Salesforce::GQUIC|Zeek::PROFINET|Zeek::S7comm|Zeek::TDS)" -ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 15 -ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-31166/detect|hassh/hassh|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" +ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(_Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|Salesforce::GQUIC|Zeek::PROFINET|Zeek::S7comm|Zeek::TDS)" +ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 16 +ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|hassh/hassh|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ cd /tmp/logs && \ diff --git a/README.md b/README.md index b4f5106cd..dfe152ec2 100644 --- a/README.md +++ b/README.md @@ -141,21 +141,21 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/arkime 3.3.0 xxxxxxxxxxxx 39 hours ago 683MB -malcolmnetsec/elasticsearch-od 3.3.0 xxxxxxxxxxxx 40 hours ago 690MB -malcolmnetsec/file-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 470MB -malcolmnetsec/file-upload 3.3.0 xxxxxxxxxxxx 39 hours ago 199MB -malcolmnetsec/filebeat-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 555MB -malcolmnetsec/freq 3.3.0 xxxxxxxxxxxx 39 hours ago 390MB -malcolmnetsec/htadmin 3.3.0 xxxxxxxxxxxx 39 hours ago 180MB -malcolmnetsec/kibana-helper 3.3.0 xxxxxxxxxxxx 40 hours ago 141MB -malcolmnetsec/kibana-od 3.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB -malcolmnetsec/logstash-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB -malcolmnetsec/name-map-ui 3.3.0 xxxxxxxxxxxx 39 hours ago 137MB -malcolmnetsec/nginx-proxy 3.3.0 xxxxxxxxxxxx 39 hours ago 120MB -malcolmnetsec/pcap-capture 3.3.0 xxxxxxxxxxxx 39 hours ago 111MB -malcolmnetsec/pcap-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 157MB -malcolmnetsec/zeek 3.3.0 xxxxxxxxxxxx 39 hours ago 887MB +malcolmnetsec/arkime 3.3.1 xxxxxxxxxxxx 39 hours ago 683MB +malcolmnetsec/elasticsearch-od 3.3.1 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 3.3.1 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 3.3.1 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 3.3.1 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 3.3.1 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 3.3.1 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-helper 3.3.1 xxxxxxxxxxxx 40 hours ago 141MB +malcolmnetsec/kibana-od 3.3.1 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 3.3.1 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/name-map-ui 3.3.1 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 3.3.1 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 3.3.1 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 3.3.1 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 3.3.1 xxxxxxxxxxxx 39 hours ago 887MB ``` #### Import from pre-packaged tarballs @@ -218,10 +218,11 @@ Malcolm leverages the following excellent open source tools, among others. * Andrew Klaus's [zeek-httpattacks](https://github.com/precurse/zeek-httpattacks) plugin for detecting noncompliant HTTP requests * ICS protocol analyzers for Zeek published by [DHS CISA](https://github.com/cisagov/ICSNPP) and [Idaho National Lab](https://github.com/idaholab/ICSNPP) * Corelight's [bro-xor-exe](https://github.com/corelight/bro-xor-exe-plugin) plugin - * Corelight's ["bad neighbor" (CVE-2020-16898)](https://github.com/corelight/CVE-2020-16898) plugin - * Corelight's [HTTP protocol stack vulnerability (CVE-2021-31166)](https://github.com/corelight/CVE-2021-31166) plugin + * Corelight's ["bad neighbor" (CVE-2020-16898)](https://github.com/corelight/CVE-2020-16898) plugin + * Corelight's [HTTP protocol stack vulnerability (CVE-2021-31166)](https://github.com/corelight/CVE-2021-31166) plugin * Corelight's [callstranger-detector](https://github.com/corelight/callstranger-detector) plugin * Corelight's [community ID](https://github.com/corelight/zeek-community-id) flow hashing plugin + * Corelight's ["OMIGOD" (CVE-2021-38647)](https://github.com/corelight/CVE-2021-38647) plugin * Corelight's [pingback](https://github.com/corelight/pingback) plugin * Corelight's [ripple20](https://github.com/corelight/ripple20) plugin * Corelight's [SIGred](https://github.com/corelight/SIGred) plugin @@ -876,7 +877,7 @@ A remote network sensor appliance can be used to monitor network traffic, captur * monitor network interfaces * capture packets to PCAP files * detect file transfers in network traffic and extract and scan those files for threats -* generate and forward Zeek logs, Arkime sessions, and other information to [Malcolm](https://github.com/cisagov/malcolm) +* generate and forward Zeek logs, Arkime sessions, and other information to [Malcolm](https://github.com/cisagov/Malcolm) Please see the [Hedgehog Linux README](https://github.com/cisagov/Malcolm/blob/main/sensor-iso/README.md) for more information. @@ -1510,7 +1511,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-3.3.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-3.3.1.iso" … ``` @@ -1893,21 +1894,21 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/arkime 3.3.0 xxxxxxxxxxxx 39 hours ago 683MB -malcolmnetsec/elasticsearch-od 3.3.0 xxxxxxxxxxxx 40 hours ago 690MB -malcolmnetsec/file-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 470MB -malcolmnetsec/file-upload 3.3.0 xxxxxxxxxxxx 39 hours ago 199MB -malcolmnetsec/filebeat-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 555MB -malcolmnetsec/freq 3.3.0 xxxxxxxxxxxx 39 hours ago 390MB -malcolmnetsec/htadmin 3.3.0 xxxxxxxxxxxx 39 hours ago 180MB -malcolmnetsec/kibana-helper 3.3.0 xxxxxxxxxxxx 40 hours ago 141MB -malcolmnetsec/kibana-od 3.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB -malcolmnetsec/logstash-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB -malcolmnetsec/name-map-ui 3.3.0 xxxxxxxxxxxx 39 hours ago 137MB -malcolmnetsec/nginx-proxy 3.3.0 xxxxxxxxxxxx 39 hours ago 120MB -malcolmnetsec/pcap-capture 3.3.0 xxxxxxxxxxxx 39 hours ago 111MB -malcolmnetsec/pcap-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 157MB -malcolmnetsec/zeek 3.3.0 xxxxxxxxxxxx 39 hours ago 887MB +malcolmnetsec/arkime 3.3.1 xxxxxxxxxxxx 39 hours ago 683MB +malcolmnetsec/elasticsearch-od 3.3.1 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 3.3.1 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 3.3.1 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 3.3.1 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 3.3.1 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 3.3.1 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-helper 3.3.1 xxxxxxxxxxxx 40 hours ago 141MB +malcolmnetsec/kibana-od 3.3.1 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 3.3.1 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/name-map-ui 3.3.1 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 3.3.1 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 3.3.1 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 3.3.1 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 3.3.1 xxxxxxxxxxxx 39 hours ago 887MB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. @@ -1987,7 +1988,7 @@ If you checked out a working copy of the Malcolm repository from GitHub with a ` ### Scenario 2: Malcolm was installed from a packaged tarball -If you installed Malcolm from [pre-packaged installation files](https://github.com/cisagov/malcolm#Packager), here are the basic steps to perform an upgrade: +If you installed Malcolm from [pre-packaged installation files](https://github.com/cisagov/Malcolm#Packager), here are the basic steps to perform an upgrade: 1. stop Malcolm * `./scripts/stop` diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 6e62c7b3d..67472a6be 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -128,7 +128,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: elasticsearch: - image: malcolmnetsec/elasticsearch-od:3.3.0 + image: malcolmnetsec/elasticsearch-od:3.3.1 restart: "no" stdin_open: false tty: true @@ -165,7 +165,7 @@ services: retries: 3 start_period: 180s kibana-helper: - image: malcolmnetsec/kibana-helper:3.3.0 + image: malcolmnetsec/kibana-helper:3.3.1 restart: "no" stdin_open: false tty: true @@ -193,7 +193,7 @@ services: retries: 3 start_period: 30s kibana: - image: malcolmnetsec/kibana-od:3.3.0 + image: malcolmnetsec/kibana-od:3.3.1 restart: "no" stdin_open: false tty: true @@ -214,7 +214,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:3.3.0 + image: malcolmnetsec/logstash-oss:3.3.1 restart: "no" stdin_open: false tty: true @@ -249,7 +249,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:3.3.0 + image: malcolmnetsec/filebeat-oss:3.3.1 restart: "no" stdin_open: false tty: true @@ -286,7 +286,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:3.3.0 + image: malcolmnetsec/arkime:3.3.1 restart: "no" stdin_open: false tty: true @@ -325,7 +325,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:3.3.0 + image: malcolmnetsec/zeek:3.3.1 restart: "no" stdin_open: false tty: true @@ -351,7 +351,7 @@ services: retries: 3 start_period: 60s file-monitor: - image: malcolmnetsec/file-monitor:3.3.0 + image: malcolmnetsec/file-monitor:3.3.1 restart: "no" stdin_open: false tty: true @@ -374,7 +374,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:3.3.0 + image: malcolmnetsec/pcap-capture:3.3.1 restart: "no" stdin_open: false tty: true @@ -394,7 +394,7 @@ services: volumes: - ./pcap/upload:/pcap pcap-monitor: - image: malcolmnetsec/pcap-monitor:3.3.0 + image: malcolmnetsec/pcap-monitor:3.3.1 restart: "no" stdin_open: false tty: true @@ -417,7 +417,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:3.3.0 + image: malcolmnetsec/file-upload:3.3.1 restart: "no" stdin_open: false tty: true @@ -443,7 +443,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:3.3.0 + image: malcolmnetsec/htadmin:3.3.1 restart: "no" stdin_open: false tty: true @@ -465,7 +465,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:3.3.0 + image: malcolmnetsec/freq:3.3.1 restart: "no" stdin_open: false tty: true @@ -483,7 +483,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:3.3.0 + image: malcolmnetsec/name-map-ui:3.3.1 restart: "no" stdin_open: false tty: true @@ -504,7 +504,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:3.3.0 + image: malcolmnetsec/nginx-proxy:3.3.1 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index d9e892f8d..ee27af114 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,7 @@ services: build: context: . dockerfile: Dockerfiles/elasticsearch.Dockerfile - image: malcolmnetsec/elasticsearch-od:3.3.0 + image: malcolmnetsec/elasticsearch-od:3.3.1 restart: "no" stdin_open: false tty: true @@ -171,7 +171,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana-helper.Dockerfile - image: malcolmnetsec/kibana-helper:3.3.0 + image: malcolmnetsec/kibana-helper:3.3.1 restart: "no" stdin_open: false tty: true @@ -202,7 +202,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana.Dockerfile - image: malcolmnetsec/kibana-od:3.3.0 + image: malcolmnetsec/kibana-od:3.3.1 restart: "no" stdin_open: false tty: true @@ -226,7 +226,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:3.3.0 + image: malcolmnetsec/logstash-oss:3.3.1 restart: "no" stdin_open: false tty: true @@ -266,7 +266,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:3.3.0 + image: malcolmnetsec/filebeat-oss:3.3.1 restart: "no" stdin_open: false tty: true @@ -307,7 +307,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:3.3.0 + image: malcolmnetsec/arkime:3.3.1 restart: "no" stdin_open: false tty: true @@ -352,7 +352,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:3.3.0 + image: malcolmnetsec/zeek:3.3.1 restart: "no" stdin_open: false tty: true @@ -382,7 +382,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:3.3.0 + image: malcolmnetsec/file-monitor:3.3.1 restart: "no" stdin_open: false tty: true @@ -408,7 +408,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:3.3.0 + image: malcolmnetsec/pcap-capture:3.3.1 restart: "no" stdin_open: false tty: true @@ -431,7 +431,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:3.3.0 + image: malcolmnetsec/pcap-monitor:3.3.1 restart: "no" stdin_open: false tty: true @@ -457,7 +457,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:3.3.0 + image: malcolmnetsec/file-upload:3.3.1 restart: "no" stdin_open: false tty: true @@ -483,7 +483,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:3.3.0 + image: malcolmnetsec/htadmin:3.3.1 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -508,7 +508,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:3.3.0 + image: malcolmnetsec/freq:3.3.1 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -529,7 +529,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:3.3.0 + image: malcolmnetsec/name-map-ui:3.3.1 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -556,7 +556,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:3.3.0 + image: malcolmnetsec/nginx-proxy:3.3.1 restart: "no" stdin_open: false tty: true diff --git a/kibana/scripts/kibana-create-moloch-sessions-index.sh b/kibana/scripts/kibana-create-moloch-sessions-index.sh index 243015153..1fce45833 100755 --- a/kibana/scripts/kibana-create-moloch-sessions-index.sh +++ b/kibana/scripts/kibana-create-moloch-sessions-index.sh @@ -128,6 +128,7 @@ if [[ "$CREATE_ES_ARKIME_SESSION_INDEX" = "true" ]] ; then # before we go on to create the anomaly detectors, we need to wait for actual sessions2-* documents /data/elastic_search_status.sh -w >/dev/null 2>&1 + sleep 60 echo "Creating Kibana anomaly detectors..." diff --git a/logstash/maps/service_ports.yaml b/logstash/maps/service_ports.yaml index 9af23e691..b3b46e58a 100644 --- a/logstash/maps/service_ports.yaml +++ b/logstash/maps/service_ports.yaml @@ -49,6 +49,8 @@ http: - 3702 - 5357 - 5358 + - 5985 + - 5986 - 8000 - 8080 - 8443 diff --git a/scripts/build.sh b/scripts/build.sh index 38e75330d..6eef28eed 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -23,7 +23,7 @@ elif $GREP -q Microsoft /proc/version && docker-compose.exe version >/dev/null 2 DOCKER_BIN=docker.exe fi -if [ "$1" ]; then +if [[ -f "$1" ]]; then CONFIG_FILE="$1" DOCKER_COMPOSE_COMMAND="$DOCKER_COMPOSE_BIN -f "$CONFIG_FILE"" shift # use remainder of arguments for services @@ -38,6 +38,13 @@ function filesize_in_image() { $DOCKER_BIN run --rm --entrypoint /bin/sh "$IMAGE" -c "stat --printf='%s' \"$FILESPEC\" 2>/dev/null || stat -c '%s' \"$FILESPEC\" 2>/dev/null" } +function dirsize_in_image() { + FILESPEC="$2" + IMAGE="$($GREP -P "^\s+image:.*$1" docker-compose-standalone.yml | awk '{print $2}')" + KBYTES="$($DOCKER_BIN run --rm --entrypoint /bin/sh "$IMAGE" -c "du -sk \"$FILESPEC\" 2>/dev/null | cut -f1")" + echo $(($KBYTES * 1024)) +} + # force-navigate to Malcolm base directory (parent of scripts/ directory) SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 @@ -93,9 +100,6 @@ fi # we're going to do some validation that some things got pulled/built correctly FILES_IN_IMAGES=( "/usr/share/filebeat/filebeat.yml;filebeat-oss" - "/var/lib/clamav/main.cvd;file-monitor" - "/var/lib/clamav/daily.cld;file-monitor" - "/var/lib/clamav/bytecode.cvd;file-monitor" "/var/www/upload/js/jquery.fileupload.js;file-upload" "/opt/freq_server/freq_server.py;freq" "/var/www/htadmin/index.php;htadmin" @@ -117,3 +121,13 @@ for i in ${FILES_IN_IMAGES[@]}; do IMAGE="$(echo "$i" | cut -d';' -f2)" (( "$(filesize_in_image $IMAGE "$FILE")" > 0 )) || { echo "Failed to create \"$FILE\" in \"$IMAGE\""; exit 1; } done + +DIRS_IN_IMAGES=( + "/var/lib/clamav;file-monitor;200000000" +) +for i in ${DIRS_IN_IMAGES[@]}; do + DIR="$(echo "$i" | cut -d';' -f1)" + IMAGE="$(echo "$i" | cut -d';' -f2)" + MINSIZE="$(echo "$i" | cut -d';' -f3)" + (( "$(dirsize_in_image $IMAGE "$DIR")" > $MINSIZE )) || { echo "Failed to create \"$DIR\" in \"$IMAGE\""; exit 1; } +done diff --git a/sensor-iso/README.md b/sensor-iso/README.md index 68278792f..d5ab079b0 100644 --- a/sensor-iso/README.md +++ b/sensor-iso/README.md @@ -8,7 +8,7 @@ Hedgehog Linux is a Debian-based operating system built to * monitor network interfaces * capture packets to PCAP files * detect file transfers in network traffic and extract and scan those files for threats -* generate and forward Zeek logs, Arkime sessions and other information to [Malcolm](https://github.com/cisagov/malcolm) +* generate and forward Zeek logs, Arkime sessions and other information to [Malcolm](https://github.com/cisagov/Malcolm) ### Table of Contents @@ -153,7 +153,7 @@ In either case, upon selecting **OK** the network interface will be brought down ### Time synchronization -Returning to the configuration mode selection, choose **Time Sync**. Here you can configure the sensor to keep its time synchronized with either an NTP server (using the NTP protocol) or a local [Malcolm](https://github.com/cisagov/malcolm) aggregator or another HTTP/HTTPS server. On the next dialog, choose the time synchronization method you wish to configure. +Returning to the configuration mode selection, choose **Time Sync**. Here you can configure the sensor to keep its time synchronized with either an NTP server (using the NTP protocol) or a local [Malcolm](https://github.com/cisagov/Malcolm) aggregator or another HTTP/HTTPS server. On the next dialog, choose the time synchronization method you wish to configure. ![Time synchronization method](./docs/images/time_sync_mode.png) @@ -210,7 +210,7 @@ You'll be prompted to specify which engine(s) to use to analyze extracted files. * scanning files with [**Yara**](https://github.com/VirusTotal/yara); to enable this method, select **ZEEK_FILE_SCAN_YARA** when specifying scanners for Zeek-carved files * scanning portable executable (PE) files with [**Capa**](https://github.com/fireeye/capa); to enable this method, select **ZEEK_FILE_SCAN_CAPA** when specifying scanners for Zeek-carved files -Files which are flagged as potentially malicious will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [Kibana](https://github.com/cisagov/malcolm#KibanaVisualizations) when forwarded to Malcolm. +Files which are flagged as potentially malicious will be logged as Zeek `signatures.log` entries, and can be viewed in the **Signatures** dashboard in [Kibana](https://github.com/cisagov/Malcolm#KibanaVisualizations) when forwarded to Malcolm. ![File quarantine](./docs/images/file_quarantine.png) @@ -220,7 +220,7 @@ Finally, you will then be presented with the list of configuration variables tha ### Forwarding -Select **Configure Forwarding** to set up forwarding logs and statistics from the sensor to an aggregator server, such as [Malcolm](https://github.com/cisagov/malcolm) or another [Elastic Stack](https://www.elastic.co/products/)-based server. +Select **Configure Forwarding** to set up forwarding logs and statistics from the sensor to an aggregator server, such as [Malcolm](https://github.com/cisagov/Malcolm) or another [Elastic Stack](https://www.elastic.co/products/)-based server. ![Configure forwarders](./docs/images/forwarder_config.png) @@ -238,7 +238,7 @@ Next you are asked whether the connection used for Zeek log forwarding should be ![Filebeat SSL certificate verification](./docs/images/filebeat_ssl.png) -If **SSL** is chosen, you must choose whether to enable [SSL certificate verification](https://www.elastic.co/guide/en/beats/filebeat/current/configuring-ssl-logstash.html). If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/malcolm#configure-authentication), choose **None**. +If **SSL** is chosen, you must choose whether to enable [SSL certificate verification](https://www.elastic.co/guide/en/beats/filebeat/current/configuring-ssl-logstash.html). If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/Malcolm#configure-authentication), choose **None**. ![Unencrypted vs. SSL encryption for Zeek log forwarding](./docs/images/filebeat_ssl_verify.png) @@ -254,9 +254,9 @@ Once you have specified all of the filebeat parameters, you will be presented wi ### moloch-capture: Arkime session forwarding -[moloch-capture](https://github.com/arkime/arkime/tree/master/capture) is not only used to capture PCAP files, but also the parse raw traffic into sessions and forward this session metadata to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) database so that it can be viewed in [Arkime viewer](https://molo.ch/), whether standalone or as part of a [Malcolm](https://github.com/cisagov/malcolm) instance. If you're using Hedgehog Linux with Malcolm, please read [Correlating Zeek logs and Arkime sessions](https://github.com/cisagov/malcolm#ZeekArkimeFlowCorrelation) in the Malcolm documentation for more information. +[moloch-capture](https://github.com/arkime/arkime/tree/master/capture) is not only used to capture PCAP files, but also the parse raw traffic into sessions and forward this session metadata to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) database so that it can be viewed in [Arkime viewer](https://molo.ch/), whether standalone or as part of a [Malcolm](https://github.com/cisagov/Malcolm) instance. If you're using Hedgehog Linux with Malcolm, please read [Correlating Zeek logs and Arkime sessions](https://github.com/cisagov/Malcolm#ZeekArkimeFlowCorrelation) in the Malcolm documentation for more information. -First, select the Elasticsearch connection transport protocol, either **HTTPS** or **HTTP**. If the metrics are being forwarded to Malcolm, select **HTTPS** to encrypt messages from the sensor to the aggregator using TLS v1.2 using ECDHE-RSA-AES128-GCM-SHA256. If **HTTPS** is chosen, you must choose whether to enable SSL certificate verification. If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/malcolm#configure-authentication)), choose **None**. +First, select the Elasticsearch connection transport protocol, either **HTTPS** or **HTTP**. If the metrics are being forwarded to Malcolm, select **HTTPS** to encrypt messages from the sensor to the aggregator using TLS v1.2 using ECDHE-RSA-AES128-GCM-SHA256. If **HTTPS** is chosen, you must choose whether to enable SSL certificate verification. If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/Malcolm#configure-authentication)), choose **None**. ![Elasticsearch connection protocol](./docs/images/metricbeat_elastic_protocol.png) ![Elasticsearch SSL verification](./docs/images/metricbeat_elastic_ssl.png) @@ -284,7 +284,7 @@ Metricbeat gathers system resource metrics at an interval you specify. The defau ![Metricbeat interval](./docs/images/metricbeat_interval.png) -Next, select the Elasticsearch connection transport protocol, either **HTTPS** or **HTTP**. If the metrics are being forwarded to Malcolm, select **HTTPS** to encrypt messages from the sensor to the aggregator using TLS v1.2 using ECDHE-RSA-AES128-GCM-SHA256. If **HTTPS** is chosen, you must choose whether to enable SSL certificate verification. If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/malcolm#configure-authentication), choose **None**. +Next, select the Elasticsearch connection transport protocol, either **HTTPS** or **HTTP**. If the metrics are being forwarded to Malcolm, select **HTTPS** to encrypt messages from the sensor to the aggregator using TLS v1.2 using ECDHE-RSA-AES128-GCM-SHA256. If **HTTPS** is chosen, you must choose whether to enable SSL certificate verification. If you are using a self-signed certificate (such as the one automatically created during [Malcolm's configuration](https://github.com/cisagov/Malcolm#configure-authentication), choose **None**. ![Elasticsearch connection protocol](./docs/images/metricbeat_elastic_protocol.png) ![Elasticsearch SSL verification](./docs/images/metricbeat_elastic_ssl.png) @@ -336,7 +336,7 @@ Despite configuring capture and/or forwarder services as described in previous s * **AUTOSTART_HEATBEAT** – [sensor hardware](#heatbeat) (eg., CPU and storage device temperature) metrics forwarder * **AUTOSTART_HEATBEAT_SENSORS** – the background process monitoring [hardware sensors](#heatbeat) for temperatures, voltages, fan speeds, etc. (this is required in addition to **AUTOSTART_HEATBEAT** metrics forwarding) * **AUTOSTART_METRICBEAT** – system resource utilization [metrics forwarder](#metricbeat) -* **AUTOSTART_ARKIME** – [moloch-capture](##moloch-capture) PCAP engine for traffic capture, as well as traffic parsing and metadata insertion into Elasticsearch for viewing in [Arkime](https://molo.ch/). If you are using Hedgehog Linux along with [Malcolm](https://github.com/cisagov/malcolm) or another Arkime installation, this is probably the packet capture engine you want to use. +* **AUTOSTART_ARKIME** – [moloch-capture](##moloch-capture) PCAP engine for traffic capture, as well as traffic parsing and metadata insertion into Elasticsearch for viewing in [Arkime](https://molo.ch/). If you are using Hedgehog Linux along with [Malcolm](https://github.com/cisagov/Malcolm) or another Arkime installation, this is probably the packet capture engine you want to use. * *AUTOSTART_NETSNIFF* – [netsniff-ng](http://netsniff-ng.org/) PCAP engine for saving packet capture (PCAP) files * **AUTOSTART_PRUNE_ZEEK** – storage space monitor to ensure that Zeek logs do not consume more than 90% of the total size of the storage volume to which Zeek logs are written * **AUTOSTART_PRUNE_PCAP** – storage space monitor to ensure that PCAP files do not consume more than 90% of the total size of the storage volume to which PCAP files are written @@ -416,7 +416,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-3.3.0.iso" +Finished, created "/sensor-build/hedgehog-3.3.1.iso" … ``` @@ -943,7 +943,7 @@ Once the Hedgehog has come back up, check to make sure everything is working: * `sensorwatch` should show current writes to Zeek log files and PCAP files (depending on your configuration) * `tail -f /opt/sensor/sensor_ctl/log/*` should show no egregious errors * `zeek --version`, `zeek -N local` and `moloch-capture --version` ought to run and print out version information as expected -* if you are forwarding to a [Malcolm](https://github.com/cisagov/malcolm) aggregator, you should start seeing data momentarily +* if you are forwarding to a [Malcolm](https://github.com/cisagov/Malcolm) aggregator, you should start seeing data momentarily # Copyright diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 44cf128f1..fdd9ae0d4 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -12,7 +12,7 @@ export CCACHE_COMPRESS=1 NETSNIFF_VER="0.6.8" NETSNIFF_URL="https://github.com/netsniff-ng/netsniff-ng/archive/v$NETSNIFF_VER.tar.gz" -ZEEK_VERSION=4.0.3-0 +ZEEK_VERSION=4.0.4-0 ZEEK_LTS=1 SPICY_VERSION=1.2.1 ZEEK_DIR="/opt/zeek" @@ -29,7 +29,7 @@ YARA_URL="https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz" YARA_RULES_URL="https://codeload.github.com/Neo23x0/signature-base/tar.gz/master" YARA_RULES_DIR="/opt/yara-rules" -CAPA_VERSION="2.0.0" +CAPA_VERSION="3.0.2" CAPA_URL="https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip" CAPA_DIR="/usr/local/bin" CAPA_BIN="${CAPA_DIR}/capa" diff --git a/shared/bin/zeek_carve_utils.py b/shared/bin/zeek_carve_utils.py index e378a44da..32854e3bd 100644 --- a/shared/bin/zeek_carve_utils.py +++ b/shared/bin/zeek_carve_utils.py @@ -98,7 +98,7 @@ CAPA_SUBMIT_TIMEOUT_SEC = 60 CAPA_ENGINE_ID = 'Capa' CAPA_CHECK_INTERVAL = 0.1 -CAPA_MIMES_TO_SCAN = ('application/bat', 'application/ecmascript', 'application/javascript', 'application/PowerShell', 'application/vnd.microsoft.portable-executable', 'application/x-bat', 'application/x-dosexec', 'application/x-executable', 'application/x-msdos-program', 'application/x-msdownload', 'application/x-pe-app-32bit-i386', 'application/x-sh', 'text/jscript', 'text/vbscript', 'text/x-python', 'text/x-shellscript') +CAPA_MIMES_TO_SCAN = ('application/bat', 'application/ecmascript', 'application/javascript', 'application/PowerShell', 'application/vnd.microsoft.portable-executable', 'application/x-bat', 'application/x-dosexec', 'application/x-elf', 'application/x-executable', 'application/x-msdos-program', 'application/x-msdownload', 'application/x-pe-app-32bit-i386', 'application/x-sh', 'text/jscript', 'text/vbscript', 'text/x-python', 'text/x-shellscript') CAPA_VIV_SUFFIX = '.viv' CAPA_VIV_MIME = 'data' CAPA_ATTACK_KEY = 'att&ck' diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index 3d009cd98..3ab2f743f 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -86,6 +86,7 @@ ZKG_GITHUB_URLS=( "https://github.com/corelight/callstranger-detector" "https://github.com/corelight/CVE-2020-16898" "https://github.com/corelight/CVE-2021-31166" + "https://github.com/corelight/CVE-2021-38647|master" "https://github.com/corelight/pingback" "https://github.com/corelight/ripple20" "https://github.com/corelight/SIGRed"