From 58d2211bf4ca8e50474f0a241d147be7a90e78ae Mon Sep 17 00:00:00 2001 From: SG <13872653+mmguero@users.noreply.github.com> Date: Thu, 25 Jul 2019 15:50:17 -0600 Subject: [PATCH] multiple user accounts and account management (#39) * bump version for next point release * increase length of self-signed cert * Prompt during installation for whether or not to disable IPv6 * In the Kibana Software dashboard pie chart, don't exclude software for which a version is unknown * Allow Malcolm to be restarted automatically upon docker daemon restart * sync malcolm and iso installer * working on using htadmin as a user management frontend * working more on account management * working on multiple users * tweak nginx according to https://github.com/aol/moloch/pull/1120 * more work on multi-user auto creation with moloch * more work on multi-user auto creation with moloch * more work on multi-user auto creation with moloch * more work on multi-user auto creation with moloch * bump version, udpate README for account management * Improved comments for zeeklogs wise data source * added user desktop file for malcolm iso * create missing htadmin directory in iso build * relocate icons for malcolm from desktop * BIOS ISO is kind of busted at the moment anyway as far as preseeding goes, so for now just install the EFI bootloader. * fix read permissions for group/world on htpasswd file for nginx processes * A couple fixes for the iso: 1) disable some callback features in chromium via policy, 2) let wicd manage networking * When running auth_setup.sh to reset the administrator password, don't blow away the other passwords in the database * working on using htadmin as a user management frontend * working on multiple users * tweak nginx according to https://github.com/aol/moloch/pull/1120 * more work on multi-user auto creation with moloch * more work on multi-user auto creation with moloch * added user desktop file for malcolm iso --- .gitignore | 2 + Dockerfiles/file-monitor.Dockerfile | 4 +- Dockerfiles/file-upload.Dockerfile | 8 +- Dockerfiles/htadmin.Dockerfile | 66 ++++++ Dockerfiles/moloch.Dockerfile | 7 +- README.md | 125 +++++----- docker-compose-standalone-zeek-live.yml | 34 ++- docker-compose-standalone.yml | 34 ++- docker-compose.yml | 37 ++- htadmin/docker-entrypoint.sh | 33 +++ htadmin/nginx/sites-available/default | 25 ++ htadmin/php/php.ini | 220 ++++++++++++++++++ htadmin/supervisord.conf | 32 +++ iso-build/build.sh | 5 +- .../skel/.config/lxpanel/LXDE/panels/malcolm | 145 ++++++++++++ .../applications/malcolm-cyberchef.desktop} | 0 .../applications/malcolm-kibana.desktop} | 0 .../share/applications/malcolm-logs.desktop} | 0 .../applications/malcolm-moloch.desktop} | 0 .../applications/malcolm-readme.desktop} | 0 .../applications/malcolm-restart.desktop} | 2 +- .../share/applications/malcolm-start.desktop} | 0 .../share/applications/malcolm-stop.desktop} | 0 .../applications/malcolm-upload.desktop} | 0 .../share/applications/malcolm-users.desktop | 11 + .../usr/share/applications/users.desktop | 11 + .../config/package-lists/system.list.chroot | 1 + moloch/etc/config.ini | 6 +- ..._creation_of_users_pull_1120_4d3f69a.patch | 89 +++++++ moloch/scripts/bs4_remove_div.py | 77 ++++++ nginx/nginx.conf | 20 +- scripts/auth_setup.sh | 46 +++- scripts/malcolm_appliance_packager.sh | 6 +- scripts/start.sh | 17 +- 34 files changed, 953 insertions(+), 110 deletions(-) create mode 100644 Dockerfiles/htadmin.Dockerfile create mode 100755 htadmin/docker-entrypoint.sh create mode 100644 htadmin/nginx/sites-available/default create mode 100644 htadmin/php/php.ini create mode 100644 htadmin/supervisord.conf create mode 100644 iso-build/config/includes.chroot/etc/skel/.config/lxpanel/LXDE/panels/malcolm rename iso-build/config/includes.chroot/{etc/skel/Desktop/cyberchef.desktop => usr/share/applications/malcolm-cyberchef.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/kibana.desktop => usr/share/applications/malcolm-kibana.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/logs.desktop => usr/share/applications/malcolm-logs.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/moloch.desktop => usr/share/applications/malcolm-moloch.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/readme.desktop => usr/share/applications/malcolm-readme.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/restart.desktop => usr/share/applications/malcolm-restart.desktop} (63%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/start.desktop => usr/share/applications/malcolm-start.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/stop.desktop => usr/share/applications/malcolm-stop.desktop} (100%) mode change 100755 => 100644 rename iso-build/config/includes.chroot/{etc/skel/Desktop/upload.desktop => usr/share/applications/malcolm-upload.desktop} (100%) mode change 100755 => 100644 create mode 100644 iso-build/config/includes.chroot/usr/share/applications/malcolm-users.desktop create mode 100755 iso-build/config/includes.chroot/usr/share/applications/users.desktop create mode 100644 moloch/patch/moloch_enable_implicit_creation_of_users_pull_1120_4d3f69a.patch create mode 100755 moloch/scripts/bs4_remove_div.py diff --git a/.gitignore b/.gitignore index 49caf8fd3..dacd12eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ /filebeat/certs /nginx/certs /nginx/htpasswd +/htadmin/config.ini +/htadmin/metadata # development .vagrant diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index f84af3089..d3912e017 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -58,8 +58,8 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list python3-requests && \ pip3 install clamd namedlist supervisor && \ mkdir -p /var/log/supervisor && \ - apt-get -y -q --force-yes --purge remove python3-dev build-essential && \ - apt-get -y -q --force-yes autoremove && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove python3-dev build-essential && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ wget -O /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \ diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile index 83f8db873..65260e4b5 100644 --- a/Dockerfiles/file-upload.Dockerfile +++ b/Dockerfiles/file-upload.Dockerfile @@ -10,7 +10,7 @@ ENV JQUERY_FILE_UPLOAD_VERSION v9.19.1 ADD https://github.com/blueimp/jQuery-File-Upload/archive/${JQUERY_FILE_UPLOAD_VERSION}.tar.gz /jQuery-File-Upload.tar.gz RUN apt-get update && \ - apt-get -y -q --force-yes install --no-install-recommends npm node-encoding git ca-certificates wget && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends npm node-encoding git ca-certificates wget && \ npm install -g bower && \ mkdir /jQuery-File-Upload && \ tar --strip-components=1 -C /jQuery-File-Upload -xzf /jQuery-File-Upload.tar.gz && \ @@ -30,9 +30,8 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm RUN apt-get update && \ - apt-get -y -q --force-yes install --no-install-recommends \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends \ wget \ - apt-transport-https \ ca-certificates \ openssh-server \ supervisor \ @@ -42,7 +41,8 @@ RUN apt-get update && \ php7.3-fpm \ php7.3-apcu \ nginx-light && \ - apt-get clean -y -q + apt-get clean -y -q && \ + rm -rf /var/lib/apt/lists/* ADD file-upload/supervisord.conf /supervisord.conf ADD file-upload/jquery-file-upload/index.html /var/www/upload/index.html diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile new file mode 100644 index 000000000..5558f97bb --- /dev/null +++ b/Dockerfiles/htadmin.Dockerfile @@ -0,0 +1,66 @@ +FROM debian:buster-slim +# Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. +LABEL maintainer="Seth.Grover@inl.gov" + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM xterm + +ARG PHP_VERSION=7.3 +ARG MCRYPT_VERSION=1.0.2 +ARG BOOTSTRAP_VERSION=3.3.6 + +ENV PHP_VERSION $PHP_VERSION +ENV MCRYPT_VERSION $MCRYPT_VERSION +ENV BOOTSTRAP_VERSION $BOOTSTRAP_VERSION + +RUN apt-get update && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends install \ + bcrypt \ + ca-certificates \ + curl \ + git \ + libmcrypt-dev \ + libmcrypt4 \ + make \ + mcrypt \ + nginx-light \ + php-dev \ + php-pear \ + php$PHP_VERSION-apcu \ + php$PHP_VERSION-cli \ + php$PHP_VERSION-curl \ + php$PHP_VERSION-fpm \ + php$PHP_VERSION-gd \ + procps \ + supervisor && \ + ( yes '' | pecl channel-update pecl.php.net ) && \ + ( yes '' | pecl install mcrypt-$MCRYPT_VERSION ) && \ + ln -s -r /usr/lib/php/20??????/*.so /usr/lib/php/$PHP_VERSION/ && \ + mkdir -p /run/php && \ + git clone --depth 1 https://github.com/mmguero/htadmin /tmp/htadmin && \ + mv /tmp/htadmin/sites/html/htadmin /var/www/htadmin && \ + cd /var/www/htadmin && \ + ( grep -rhoPi "(src|href)=['\"]https?://.+?['\"]" ./includes/* | sed "s/^[a-zA-Z]*=['\"]*//" | sed "s/['\"]$//" | xargs -r -l curl -s -S -L -J -O ) && \ + sed -i "s@http[^'\"]*/@@gI" ./includes/* && \ + mkdir fonts && cd fonts && \ + curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.ttf" && \ + curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff" && \ + curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff2" && \ + cd /tmp && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \ + git make libmcrypt-dev php-pear php-dev && \ + apt-get autoremove -y -q && \ + apt-get clean -y -q && \ + usermod --non-unique --uid 1000 www-data && \ + groupmod --non-unique --gid 1000 www-data && \ + chown -R www-data:www-data /var/www && \ + rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /var/tmp/* /var/www/html + +ADD docs/images/favicon/favicon.ico /var/www/htadmin/ +ADD htadmin/supervisord.conf /supervisord.conf +ADD htadmin/php/php.ini /etc/php/$PHP_VERSION/fpm/php.ini +ADD htadmin/nginx/sites-available/default /etc/nginx/sites-available/default + +EXPOSE 80 + +CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf", "-u", "root", "-n"] diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile index 82061b042..efddae395 100644 --- a/Dockerfiles/moloch.Dockerfile +++ b/Dockerfiles/moloch.Dockerfile @@ -11,6 +11,7 @@ ENV ZEEK_VERSION "2.6.2" ENV ZEEK_DIR "/opt/bro" ENV CYBERCHEF_VERSION "8.30.1" +ADD moloch/scripts/bs4_remove_div.py /data/ ADD moloch/patch/* /data/patches/ ADD README.md $MOLOCHDIR/doc/ ADD doc.css $MOLOCHDIR/doc/ @@ -49,11 +50,13 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list patch \ python-dev \ python3-dev \ + python3-pip \ rename \ sudo \ swig \ wget \ zlib1g-dev && \ + pip3 install --no-cache-dir beautifulsoup4 && \ cd /data && \ tar -xvf "bro.tar.gz" && \ rm -f "bro.tar.gz" && \ @@ -101,6 +104,8 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \ ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \ ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \ + python3 /data/bs4_remove_div.py -i ./viewer/vueapp/src/components/users/Users.vue -o ./viewer/vueapp/src/components/users/Users.new -c "new-user-form" && \ + mv -vf ./viewer/vueapp/src/components/users/Users.new ./viewer/vueapp/src/components/users/Users.vue && \ ./easybutton-build.sh --install && \ npm cache clean --force && \ apt-get clean && \ @@ -195,7 +200,7 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list vim-tiny \ wget \ tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \ - pip3 install --no-cache-dir elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \ + pip3 install --no-cache-dir beautifulsoup4 elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \ ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \ ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \ ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \ diff --git a/README.md b/README.md index b359b333b..9cd313df0 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ In short, Malcolm provides an easily deployable network analysis tool suite for * [Windows host system configuration](#HostSystemConfigWindows) * [Running Malcolm](#Running) * [Configure authentication](#AuthSetup) + * [Account management](#AccountManagement) * [Starting Malcolm](#Starting) * [Stopping and restarting Malcolm](#StopAndRestart) * [Clearing Malcolm's data](#Wipe) @@ -101,6 +102,7 @@ Pulling moloch ... done Pulling file-monitor ... done Pulling pcap-capture ... done Pulling upload ... done +Pulling htadmin ... done Pulling nginx-proxy ... done ``` @@ -108,17 +110,18 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/nginx-proxy 1.3.2 xxxxxxxxxxxx 16 hours ago 53MB -malcolmnetsec/file-upload 1.3.2 xxxxxxxxxxxx 16 hours ago 214MB -malcolmnetsec/pcap-capture 1.3.2 xxxxxxxxxxxx 17 hours ago 111MB -malcolmnetsec/file-monitor 1.3.2 xxxxxxxxxxxx 17 hours ago 353MB -malcolmnetsec/moloch 1.3.2 xxxxxxxxxxxx 17 hours ago 1.04GB -malcolmnetsec/filebeat-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 454MB -malcolmnetsec/curator 1.3.2 xxxxxxxxxxxx 17 hours ago 303MB -malcolmnetsec/logstash-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 1.14GB -malcolmnetsec/elastalert 1.3.2 xxxxxxxxxxxx 17 hours ago 268MB -malcolmnetsec/kibana-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 850MB -docker.elastic.co/elasticsearch/elasticsearch-oss 6.8.1 xxxxxxxxxxxx 3 weeks ago 765MB +malcolmnetsec/moloch 1.4.0 xxxxxxxxxxxx 27 minutes ago 517MB +malcolmnetsec/htadmin 1.4.0 xxxxxxxxxxxx 2 hours ago 180MB +malcolmnetsec/nginx-proxy 1.4.0 xxxxxxxxxxxx 4 hours ago 53MB +malcolmnetsec/file-upload 1.4.0 xxxxxxxxxxxx 24 hours ago 198MB +malcolmnetsec/pcap-capture 1.4.0 xxxxxxxxxxxx 24 hours ago 111MB +malcolmnetsec/file-monitor 1.4.0 xxxxxxxxxxxx 24 hours ago 355MB +malcolmnetsec/logstash-oss 1.4.0 xxxxxxxxxxxx 25 hours ago 1.24GB +malcolmnetsec/curator 1.4.0 xxxxxxxxxxxx 25 hours ago 303MB +malcolmnetsec/kibana-oss 1.4.0 xxxxxxxxxxxx 33 hours ago 944MB +malcolmnetsec/filebeat-oss 1.4.0 xxxxxxxxxxxx 11 days ago 459MB +malcolmnetsec/elastalert 1.4.0 xxxxxxxxxxxx 11 days ago 276MB +docker.elastic.co/elasticsearch/elasticsearch-oss 6.8.1 xxxxxxxxxxxx 5 weeks ago 769MB ``` You must run [`auth_setup.sh`](#AuthSetup) prior to running `docker-compose pull`. You should also ensure your system configuration and `docker-compose.yml` settings are tuned by running `./scripts/install.py` or `./scripts/install.py --configure` (see [System configuration and tuning](#ConfigAndTuning)). @@ -136,10 +139,11 @@ instance, wipe the database and restore Malcolm to a fresh state, etc. A few minutes after starting Malcolm (probably 5 to 10 minutes for Logstash to be completely up, depending on the system), the following services will be accessible: -* Moloch: [https://localhost](https://localhost) +* Moloch: [https://localhost:443](https://localhost:443) * Kibana: [https://localhost:5601](https://localhost:5601) * Capture File and Log Archive Upload (Web): [https://localhost:8443](https://localhost:8443) * Capture File and Log Archive Upload (SFTP): `sftp://@127.0.0.1:8022/files` +* Account Management: [https://localhost:488](https://localhost:488) ## Overview @@ -184,6 +188,7 @@ Checking out the [Malcolm source code](https://github.com/idaholab/malcolm) resu * `filebeat` - code and configuration for the `filebeat` container which ingests Zeek logs and forwards them to the `logstash` container * `file-monitor` - code and configuration for the `file-monitor` container which can scan files extracted by Zeek * `file-upload` - code and configuration for the `upload` container which serves a web browser-based upload form for uploading PCAP files and Zeek logs, and which serves an SFTP share as an alternate method for upload +* `htadmin` - configuration for the `htadmin` user account management container * `iso-build` - code and configuration for building an installer ISO for a minimal Debian-based Linux installation for running Malcolm * `kibana` - code and configuration for the `kibana` container for creating additional ad-hoc visualizations and dashboards beyond that which is provided by Moloch Viewer * `logstash` - code and configuration for the `logstash` container which parses Zeek logs and forwards them to the `elasticsearch` container @@ -199,7 +204,7 @@ Checking out the [Malcolm source code](https://github.com/idaholab/malcolm) resu and the following files of special note: -* `auth.env` - the script `./scripts/auth_setup.sh` prompts the user for the username and password to be used when configuring the HTTPS and SFTP servers served by an instance of the Malcolm appliance, and `auth.env` is the environment file where those values are stored +* `auth.env` - the script `./scripts/auth_setup.sh` prompts the user for the administrator credentials used by the Malcolm appliance, and `auth.env` is the environment file where those values are stored * `cidr-map.txt` - specify custom IP address to network segment mapping * `host-map.txt` - specify custom IP and/or MAC address to host mapping * `docker-compose.yml` - the configuration file used by `docker-compose` to build, start, and stop an instance of the Malcolm appliance @@ -218,9 +223,10 @@ Then, go take a walk or something since it will be a while. When you're done, yo * `malcolmnetsec/curator` (based on `debian:buster-slim`) * `malcolmnetsec/elastalert` (based on `bitsensor/elastalert`) -* `malcolmnetsec/filebeat-oss` (based on `docker.elastic.co/beats/filebeat-oss`) * `malcolmnetsec/file-monitor` (based on `debian:buster-slim`) * `malcolmnetsec/file-upload` (based on `debian:buster-slim`) +* `malcolmnetsec/filebeat-oss` (based on `docker.elastic.co/beats/filebeat-oss`) +* `malcolmnetsec/htadmin` (based on `debian:buster-slim`) * `malcolmnetsec/kibana-oss` (based on `docker.elastic.co/kibana/kibana-oss`) * `malcolmnetsec/logstash-oss` (based on `centos:7`) * `malcolmnetsec/moloch` (based on `debian:stretch-slim`) @@ -240,7 +246,7 @@ Additionally, the command will pull from Docker Hub: ``` $ ./scripts/malcolm_appliance_packager.sh You must set a username and password for Malcolm, and self-signed X.509 certificates will be generated -Username: analyst +Administrator username: analyst analyst password: analyst password (again): @@ -276,6 +282,7 @@ A minute or so after starting Malcolm, the following services will be accessible - Kibana: https://localhost:5601/ - PCAP Upload (web): https://localhost:8443/ - PCAP Upload (sftp): sftp://USERNAME@127.0.0.1:8022/files/ + - Account management: https://localhost:488/ ``` The above example will result in the following artifacts for distribution as explained in the script's output: @@ -528,7 +535,7 @@ As the author supposes that the target audience of this document are more likely Run `./scripts/auth_setup.sh` before starting Malcolm for the first time in order to: -* define the username and password to be used when accessing the HTTPS and SFTP interfaces provided by the local instance of the Malcolm appliance +* define the administrator account username and password * specify whether or not to (re)generate the self-signed certificates used for HTTPS access * key and certificate files are located in the `nginx/certs/` directory * specify whether or not to (re)generate the self-signed certificates used by a remote log forwarder (see the `BEATS_SSL` environment variable above) @@ -537,6 +544,14 @@ Run `./scripts/auth_setup.sh` before starting Malcolm for the first time in orde * specify whether or not to store the username/password for forwarding Logstash events to a secondary, external Elasticsearch instance (see the `ES_EXTERNAL_HOSTS`, `ES_EXTERNAL_SSL`, and `ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION` environment variables above) * these parameters are stored securely in the Logstash keystore file `logstash/certs/logstash.keystore` +#### Account management + +[`auth_setup.sh`](#AuthSetup) is used to define the username and password for the administrator account. Once Malcolm is running, the administrator account can be used to manage other user accounts via a **Malcolm User Management** page served over HTTPS on port 488 (eg., [https://localhost:488](https://localhost:488) if you are connecting locally). + +Malcolm user accounts can be used to access the [interfaces](#UserInterfaceURLs) of all of its [components](#Components), including Moloch. Moloch uses its own internal database of user accounts, so when a Malcolm user account logs in to Moloch for the first time Malcolm creates a corresponding Moloch user account automatically. This being the case, it is *not* recommended to use the Moloch **Users** settings page or change the password via the **Password** form under the Moloch **Settings** page, as those settings would not be consistently used across Malcolm. + +Users may change their passwords via the **Malcolm User Management** page by clicking **User Self Service**. A forgotten password can also be reset via an emailed link, though this requires SMTP server settings to be specified in `htadmin/config.ini` in the Malcolm installation directory. + ### Starting Malcolm [Docker compose](https://docs.docker.com/compose/) is used to coordinate running the Docker containers. To start Malcolm, navigate to the directory containing `docker-compose.yml` and run: @@ -816,12 +831,6 @@ See Moloch's usage documentation for more information on [settings](https://loca ![Moloch custom view management](./docs/images/screenshots/moloch_view_settings.png) -#### User management - -Currently, when [`auth_setup.sh`](#AuthSetup) is run Malcolm creates a single username/password used by all of its [components](#Components), and those credentials are used to authenticate across all of those components. This being the case, it is currently *not* recommended to use the Moloch **Users** settings page or change the password via the **Password** form under the Moloch **Settings** page, as these settings will not be consistently used across Malcolm. - -Multi-user management -- most likely with integration to an authentication server such as Active Directory -- is on Malcolm's roadmap for development in the (hopefully) not-too-distant future. - ## Kibana While Moloch provides very nice visualizations, especially for network traffic, [Kibana](https://www.elastic.co/guide/en/kibana/current/getting-started.html) (an open source general-purpose data visualization tool for Elasticsearch) can be used to create custom visualizations (tables, charts, graphs, dashboards, etc.) using the same data. @@ -1233,21 +1242,23 @@ Pulling moloch ... done Pulling file-monitor ... done Pulling pcap-capture ... done Pulling upload ... done +Pulling htadmin ... done Pulling nginx-proxy ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/nginx-proxy 1.3.2 xxxxxxxxxxxx 16 hours ago 53MB -malcolmnetsec/file-upload 1.3.2 xxxxxxxxxxxx 16 hours ago 214MB -malcolmnetsec/pcap-capture 1.3.2 xxxxxxxxxxxx 17 hours ago 111MB -malcolmnetsec/file-monitor 1.3.2 xxxxxxxxxxxx 17 hours ago 353MB -malcolmnetsec/curator 1.3.2 xxxxxxxxxxxx 17 hours ago 303MB -malcolmnetsec/moloch 1.3.2 xxxxxxxxxxxx 17 hours ago 1.04GB -malcolmnetsec/filebeat-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 454MB -malcolmnetsec/logstash-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 1.14GB -malcolmnetsec/elastalert 1.3.2 xxxxxxxxxxxx 17 hours ago 268MB -malcolmnetsec/kibana-oss 1.3.2 xxxxxxxxxxxx 17 hours ago 850MB -docker.elastic.co/elasticsearch/elasticsearch-oss 6.8.1 xxxxxxxxxxxx 3 weeks ago 765MB +malcolmnetsec/moloch 1.4.0 xxxxxxxxxxxx 27 minutes ago 517MB +malcolmnetsec/htadmin 1.4.0 xxxxxxxxxxxx 2 hours ago 180MB +malcolmnetsec/nginx-proxy 1.4.0 xxxxxxxxxxxx 4 hours ago 53MB +malcolmnetsec/file-upload 1.4.0 xxxxxxxxxxxx 24 hours ago 198MB +malcolmnetsec/pcap-capture 1.4.0 xxxxxxxxxxxx 24 hours ago 111MB +malcolmnetsec/file-monitor 1.4.0 xxxxxxxxxxxx 24 hours ago 355MB +malcolmnetsec/logstash-oss 1.4.0 xxxxxxxxxxxx 25 hours ago 1.24GB +malcolmnetsec/curator 1.4.0 xxxxxxxxxxxx 25 hours ago 303MB +malcolmnetsec/kibana-oss 1.4.0 xxxxxxxxxxxx 33 hours ago 944MB +malcolmnetsec/filebeat-oss 1.4.0 xxxxxxxxxxxx 11 days ago 459MB +malcolmnetsec/elastalert 1.4.0 xxxxxxxxxxxx 11 days ago 276MB +docker.elastic.co/elasticsearch/elasticsearch-oss 6.8.1 xxxxxxxxxxxx 5 weeks ago 769MB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. @@ -1255,41 +1266,45 @@ Finally, we can start Malcolm. When Malcolm starts it will stream informational user@host:~/Malcolm$ ./scripts/start.sh Creating network "malcolm_default" with the default driver Creating malcolm_file-monitor_1 ... done -Creating malcolm_pcap-capture_1 ... done +Creating malcolm_htadmin_1 ... done Creating malcolm_elasticsearch_1 ... done -Creating malcolm_moloch_1 ... done -Creating malcolm_elastalert_1 ... done +Creating malcolm_pcap-capture_1 ... done +Creating malcolm_curator_1 ... done Creating malcolm_logstash_1 ... done +Creating malcolm_elastalert_1 ... done Creating malcolm_kibana_1 ... done -Creating malcolm_upload_1 ... done +Creating malcolm_moloch_1 ... done Creating malcolm_filebeat_1 ... done +Creating malcolm_upload_1 ... done Creating malcolm_nginx-proxy_1 ... done Malcolm started, setting "INITIALIZEDB=false" in "docker-compose.yml" for subsequent runs. In a few minutes, Malcolm services will be accessible via the following URLs: ------------------------------------------------------------------------------ - - Moloch: https://localhost/ + - Moloch: https://localhost:443/ - Kibana: https://localhost:5601/ - PCAP Upload (web): https://localhost:8443/ - PCAP Upload (sftp): sftp://username@127.0.0.1:8022/files/ - - Name Command State Ports --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------$ -malcolm_elastalert_1 /usr/local/bin/elastalert- ... Up (health: starting) 3030/tcp -malcolm_elasticsearch_1 /usr/local/bin/docker-entr ... Up (health: starting) 9200/tcp, 9300/tcp -malcolm_file-monitor_1 /usr/local/bin/supervisord ... Up 3310/tcp -malcolm_filebeat_1 /usr/local/bin/docker-entr ... Up -malcolm_kibana_1 /usr/bin/supervisord -c /e ... Up (health: starting) 28991/tcp, 5601/tcp -malcolm_logstash_1 /usr/local/bin/logstash-st ... Up (health: starting) 5000/tcp, 5044/tcp, 9600/tcp -malcolm_moloch_1 /usr/bin/supervisord -c /e ... Up 8000/tcp, 8005/tcp, 8081/tcp -malcolm_nginx-proxy_1 /app/docker-entrypoint.sh ... Up 0.0.0.0:28991->28991/tcp, 0.0.0.0:3030->3030/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:5601->5601/tcp, 80/tcp, - 0.0.0.0:8443->8443/tcp, 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp -malcolm_pcap-capture_1 /usr/local/bin/supervisor.sh Up -malcolm_upload_1 /docker-entrypoint.sh /usr ... Up 127.0.0.1:8022->22/tcp, 80/tcp - -Attaching to malcolm_nginx-proxy_1, malcolm_filebeat_1, malcolm_upload_1, malcolm_kibana_1, malcolm_logstash_1, malcolm_elastalert_1, malcolm_moloch_1, malcolm_elasticsearch_1, malcolm_file-monitor_1$ - malcolm_pcap-capture_1 + - Account management: https://localhost:488/ + + Name Command State Ports +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +malcolm_curator_1 /usr/local/bin/cron_env_deb.sh Up +malcolm_elastalert_1 /usr/local/bin/elastalert- ... Up (health: starting) 3030/tcp, 3333/tcp +malcolm_elasticsearch_1 /usr/local/bin/docker-entr ... Up (health: starting) 9200/tcp, 9300/tcp +malcolm_file-monitor_1 /usr/local/bin/supervisord ... Up 3310/tcp +malcolm_filebeat_1 /usr/local/bin/docker-entr ... Up +malcolm_htadmin_1 /usr/bin/supervisord -c /s ... Up 80/tcp +malcolm_kibana_1 /usr/bin/supervisord -c /e ... Up (health: starting) 28991/tcp, 5601/tcp +malcolm_logstash_1 /usr/local/bin/logstash-st ... Up (health: starting) 5000/tcp, 5044/tcp, 9600/tcp +malcolm_moloch_1 /usr/bin/supervisord -c /e ... Up 8000/tcp, 8005/tcp, 8081/tcp +malcolm_nginx-proxy_1 /app/docker-entrypoint.sh ... Up 0.0.0.0:28991->28991/tcp, 0.0.0.0:3030->3030/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:488->488/tcp, 0.0.0.0:5601->5601/tcp, 80/tcp, + 0.0.0.0:8443->8443/tcp, 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp +malcolm_pcap-capture_1 /usr/local/bin/supervisor.sh Up +malcolm_upload_1 /docker-entrypoint.sh /usr ... Up 127.0.0.1:8022->22/tcp, 80/tcp + +Attaching to malcolm_nginx-proxy_1, malcolm_upload_1, malcolm_filebeat_1, malcolm_kibana_1, malcolm_moloch_1, malcolm_elastalert_1, malcolm_logstash_1, malcolm_curator_1, malcolm_elasticsearch_1, malcolm_htadmin_1, malcolm_pcap-capture_1, malcolm_file-monitor_1 … ``` diff --git a/docker-compose-standalone-zeek-live.yml b/docker-compose-standalone-zeek-live.yml index ea5f8d77b..287b4c2ca 100644 --- a/docker-compose-standalone-zeek-live.yml +++ b/docker-compose-standalone-zeek-live.yml @@ -105,7 +105,7 @@ services: - ./elasticsearch:/usr/share/elasticsearch/data:delegated - ./elasticsearch-backup:/opt/elasticsearch/backup:delegated kibana: - image: malcolmnetsec/kibana-oss:1.3.2 + image: malcolmnetsec/kibana-oss:1.4.0 restart: "no" hostname: kibana environment: @@ -128,7 +128,7 @@ services: retries: 3 start_period: 200s elastalert: - image: malcolmnetsec/elastalert:1.3.2 + image: malcolmnetsec/elastalert:1.4.0 restart: "no" hostname: elastalert environment: @@ -152,7 +152,7 @@ services: - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json - ./elastalert/rules/:/opt/elastalert/rules/ curator: - image: malcolmnetsec/curator:1.3.2 + image: malcolmnetsec/curator:1.4.0 restart: "no" hostname: curator environment: @@ -162,7 +162,7 @@ services: depends_on: - elasticsearch logstash: - image: malcolmnetsec/logstash-oss:1.3.2 + image: malcolmnetsec/logstash-oss:1.4.0 restart: "no" hostname: logstash environment: @@ -190,7 +190,7 @@ services: - ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro - ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro filebeat: - image: malcolmnetsec/filebeat-oss:1.3.2 + image: malcolmnetsec/filebeat-oss:1.4.0 restart: "no" hostname: filebeat environment: @@ -215,7 +215,7 @@ services: - ./filebeat/certs/client.crt:/certs/client.crt:ro - ./filebeat/certs/client.key:/certs/client.key:ro moloch: - image: malcolmnetsec/moloch:1.3.2 + image: malcolmnetsec/moloch:1.4.0 restart: "no" hostname: moloch env_file: @@ -248,7 +248,7 @@ services: - ./moloch-logs:/data/moloch/logs - ./moloch-raw:/data/moloch/raw file-monitor: - image: malcolmnetsec/file-monitor:1.3.2 + image: malcolmnetsec/file-monitor:1.4.0 restart: "no" hostname: filemon environment: @@ -259,7 +259,7 @@ services: - ./zeek-logs/extract_files:/data/zeek/extract_files - ./zeek-logs/current:/data/zeek/logs pcap-capture: - image: malcolmnetsec/pcap-capture:1.3.2 + image: malcolmnetsec/pcap-capture:1.4.0 restart: "no" network_mode: host ulimits: @@ -276,7 +276,7 @@ services: volumes: - ./pcap/upload:/pcap upload: - image: malcolmnetsec/file-upload:1.3.2 + image: malcolmnetsec/file-upload:1.4.0 restart: "no" hostname: upload env_file: @@ -292,16 +292,30 @@ services: - 127.0.0.1:8022:22 volumes: - ./pcap/upload:/var/www/upload/server/php/chroot/files + htadmin: + image: malcolmnetsec/htadmin:1.4.0 + restart: "no" + hostname: htadmin + environment: + VIRTUAL_HOST : 'htadmin.malcolm.local' + expose: + - 80 + volumes: + - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw + - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw + - ./nginx/htpasswd:/var/www/htadmin/config/htpasswd:rw nginx-proxy: - image: malcolmnetsec/nginx-proxy:1.3.2 + image: malcolmnetsec/nginx-proxy:1.4.0 restart: "no" hostname: nginx-proxy depends_on: - moloch - kibana - upload + - htadmin ports: - "443:443" + - "488:488" - "3030:3030" - "5601:5601" - "8443:8443" diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 1c1a6534e..5fd897641 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -105,7 +105,7 @@ services: - ./elasticsearch:/usr/share/elasticsearch/data:delegated - ./elasticsearch-backup:/opt/elasticsearch/backup:delegated kibana: - image: malcolmnetsec/kibana-oss:1.3.2 + image: malcolmnetsec/kibana-oss:1.4.0 restart: "no" hostname: kibana environment: @@ -128,7 +128,7 @@ services: retries: 3 start_period: 200s elastalert: - image: malcolmnetsec/elastalert:1.3.2 + image: malcolmnetsec/elastalert:1.4.0 restart: "no" hostname: elastalert environment: @@ -152,7 +152,7 @@ services: - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json - ./elastalert/rules/:/opt/elastalert/rules/ curator: - image: malcolmnetsec/curator:1.3.2 + image: malcolmnetsec/curator:1.4.0 restart: "no" hostname: curator environment: @@ -162,7 +162,7 @@ services: depends_on: - elasticsearch logstash: - image: malcolmnetsec/logstash-oss:1.3.2 + image: malcolmnetsec/logstash-oss:1.4.0 restart: "no" hostname: logstash environment: @@ -190,7 +190,7 @@ services: - ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro - ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro filebeat: - image: malcolmnetsec/filebeat-oss:1.3.2 + image: malcolmnetsec/filebeat-oss:1.4.0 restart: "no" hostname: filebeat environment: @@ -215,7 +215,7 @@ services: - ./filebeat/certs/client.crt:/certs/client.crt:ro - ./filebeat/certs/client.key:/certs/client.key:ro moloch: - image: malcolmnetsec/moloch:1.3.2 + image: malcolmnetsec/moloch:1.4.0 restart: "no" hostname: moloch env_file: @@ -248,7 +248,7 @@ services: - ./moloch-logs:/data/moloch/logs - ./moloch-raw:/data/moloch/raw file-monitor: - image: malcolmnetsec/file-monitor:1.3.2 + image: malcolmnetsec/file-monitor:1.4.0 restart: "no" hostname: filemon environment: @@ -259,7 +259,7 @@ services: - ./zeek-logs/extract_files:/data/zeek/extract_files - ./zeek-logs/current:/data/zeek/logs pcap-capture: - image: malcolmnetsec/pcap-capture:1.3.2 + image: malcolmnetsec/pcap-capture:1.4.0 restart: "no" network_mode: host ulimits: @@ -276,7 +276,7 @@ services: volumes: - ./pcap/upload:/pcap upload: - image: malcolmnetsec/file-upload:1.3.2 + image: malcolmnetsec/file-upload:1.4.0 restart: "no" hostname: upload env_file: @@ -292,16 +292,30 @@ services: - 127.0.0.1:8022:22 volumes: - ./pcap/upload:/var/www/upload/server/php/chroot/files + htadmin: + image: malcolmnetsec/htadmin:1.4.0 + restart: "no" + hostname: htadmin + environment: + VIRTUAL_HOST : 'htadmin.malcolm.local' + expose: + - 80 + volumes: + - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw + - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw + - ./nginx/htpasswd:/var/www/htadmin/config/htpasswd:rw nginx-proxy: - image: malcolmnetsec/nginx-proxy:1.3.2 + image: malcolmnetsec/nginx-proxy:1.4.0 restart: "no" hostname: nginx-proxy depends_on: - moloch - kibana - upload + - htadmin ports: - "443:443" + - "488:488" - "3030:3030" - "5601:5601" - "8443:8443" diff --git a/docker-compose.yml b/docker-compose.yml index 873850f41..982315d5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,7 +108,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana.Dockerfile - image: malcolmnetsec/kibana-oss:1.3.2 + image: malcolmnetsec/kibana-oss:1.4.0 restart: "no" hostname: kibana environment: @@ -134,7 +134,7 @@ services: build: context: . dockerfile: Dockerfiles/elastalert.Dockerfile - image: malcolmnetsec/elastalert:1.3.2 + image: malcolmnetsec/elastalert:1.4.0 restart: "no" hostname: elastalert environment: @@ -161,7 +161,7 @@ services: build: context: . dockerfile: Dockerfiles/curator.Dockerfile - image: malcolmnetsec/curator:1.3.2 + image: malcolmnetsec/curator:1.4.0 restart: "no" hostname: curator environment: @@ -176,7 +176,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:1.3.2 + image: malcolmnetsec/logstash-oss:1.4.0 restart: "no" hostname: logstash environment: @@ -214,7 +214,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:1.3.2 + image: malcolmnetsec/filebeat-oss:1.4.0 restart: "no" hostname: filebeat environment: @@ -243,7 +243,7 @@ services: build: context: . dockerfile: Dockerfiles/moloch.Dockerfile - image: malcolmnetsec/moloch:1.3.2 + image: malcolmnetsec/moloch:1.4.0 restart: "no" hostname: moloch env_file: @@ -282,7 +282,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:1.3.2 + image: malcolmnetsec/file-monitor:1.4.0 restart: "no" hostname: filemon environment: @@ -296,7 +296,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:1.3.2 + image: malcolmnetsec/pcap-capture:1.4.0 restart: "no" network_mode: host ulimits: @@ -316,7 +316,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:1.3.2 + image: malcolmnetsec/file-upload:1.4.0 restart: "no" hostname: upload env_file: @@ -332,19 +332,36 @@ services: - 127.0.0.1:8022:22 volumes: - ./pcap/upload:/var/www/upload/server/php/chroot/files + htadmin: + image: malcolmnetsec/htadmin:1.4.0 + build: + context: . + dockerfile: Dockerfiles/htadmin.Dockerfile + restart: "no" + hostname: htadmin + environment: + VIRTUAL_HOST : 'htadmin.malcolm.local' + expose: + - 80 + volumes: + - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw + - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw + - ./nginx/htpasswd:/var/www/htadmin/config/htpasswd:rw nginx-proxy: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:1.3.2 + image: malcolmnetsec/nginx-proxy:1.4.0 restart: "no" hostname: nginx-proxy depends_on: - moloch - kibana - upload + - htadmin ports: - "443:443" + - "488:488" - "3030:3030" - "5601:5601" - "8443:8443" diff --git a/htadmin/docker-entrypoint.sh b/htadmin/docker-entrypoint.sh new file mode 100755 index 000000000..b15497a7d --- /dev/null +++ b/htadmin/docker-entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. + + +if [[ -z $SITE_NAME || -z $MALCOLM_USERNAME || -z $MALCOLM_PASSWORD ]] +then + echo "Please set the site name, username and (openssl-encrypted) password by adding the following arguments to docker run/create:" + echo " -e SITE_NAME='...'" + echo " -e MALCOLM_USERNAME='...'" + echo " -e MALCOLM_PASSWORD='...'" + exit 1 +fi + +if ! getent passwd "$MALCOLM_USERNAME" >/dev/null +then + # Make sure every container gets its own SSH host keys the first time around + rm -f /etc/ssh/ssh_host_* + dpkg-reconfigure openssh-server + + useradd -g www-data -d /var/www/upload/server/php/chroot -s /sbin/nologin "$MALCOLM_USERNAME" + usermod --password "$MALCOLM_PASSWORD" "$MALCOLM_USERNAME" + chown "$MALCOLM_USERNAME:www-data" /var/www/upload/server/php/chroot/files + chmod 775 /var/www/upload/server/php/chroot/files + + # This will break if $SITE_NAME contains a slash... + sed -i 's/%SITE_NAME%/'"$SITE_NAME"'/g' /var/www/upload/index.html + +else + echo "skipping one-time setup tasks" 1>&2 +fi + +exec "$@" diff --git a/htadmin/nginx/sites-available/default b/htadmin/nginx/sites-available/default new file mode 100644 index 000000000..07a81eae0 --- /dev/null +++ b/htadmin/nginx/sites-available/default @@ -0,0 +1,25 @@ +server { + listen 80 default_server; + + sendfile on; + + root /var/www/htadmin; + index index.php index.html index.htm; + + server_name htaccess.malcolm.local; + + location / { + try_files $uri $uri/ =404; + } + + location ~ \.php$ { + include snippets/fastcgi-php.conf; + fastcgi_read_timeout 300; + fastcgi_pass unix:/run/php/php7.3-fpm.sock; + } + + location /config { + deny all; + return 404; + } +} diff --git a/htadmin/php/php.ini b/htadmin/php/php.ini new file mode 100644 index 000000000..7951b313d --- /dev/null +++ b/htadmin/php/php.ini @@ -0,0 +1,220 @@ +[PHP] + +; about php.ini +; see https://secure.php.net/manual/en/configuration.file.php +; https://secure.php.net/manual/en/ini.list.php + +engine = On +short_open_tag = Off +asp_tags = Off +precision = 14 +output_buffering = 4096 +zlib.output_compression = Off +implicit_flush = Off +unserialize_callback_func = +serialize_precision = 17 +disable_functions = pcntl_alarm,pcntl_fork,pcntl_waitpid,pcntl_wait,pcntl_wifexited,pcntl_wifstopped,pcntl_wifsignaled,pcntl_wexitstatus,pcntl_wtermsig,pcntl_wstopsig,pcntl_signal,pcntl_signal_dispatch,pcntl_get_last_error,pcntl_strerror,pcntl_sigprocmask,pcntl_sigwaitinfo,pcntl_sigtimedwait,pcntl_exec,pcntl_getpriority,pcntl_setpriority, +disable_classes = +zend.enable_gc = On +expose_php = Off +max_execution_time = 600 +max_input_time = 600 +memory_limit = 128M +error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT +display_errors = Off +display_startup_errors = Off +log_errors = On +log_errors_max_len = 1024 +ignore_repeated_errors = Off +ignore_repeated_source = Off +report_memleaks = On +track_errors = Off +html_errors = On +variables_order = "GPCS" +request_order = "GP" +register_argc_argv = Off +auto_globals_jit = On +post_max_size = 0 +auto_prepend_file = +auto_append_file = +default_mimetype = "text/html" +default_charset = "UTF-8" +doc_root = +user_dir = +enable_dl = Off +file_uploads = Off +upload_max_filesize = 500M +max_file_uploads = 4 +allow_url_fopen = On +allow_url_include = Off +default_socket_timeout = 60 +extension=mcrypt.so + +[CLI Server] +cli_server.color = On + +[Date] + +[filter] + +[iconv] + +[intl] + +[sqlite] + +[sqlite3] + +[Pcre] + +[Pdo] + +[Pdo_mysql] +pdo_mysql.cache_size = 2000 +pdo_mysql.default_socket= + +[Phar] + +[mail function] +SMTP = localhost +smtp_port = 25 +mail.add_x_header = On + +[SQL] +sql.safe_mode = Off + +[ODBC] +odbc.allow_persistent = On +odbc.check_persistent = On +odbc.max_persistent = -1 +odbc.max_links = -1 +odbc.defaultlrl = 4096 +odbc.defaultbinmode = 1 + +[Interbase] +ibase.allow_persistent = 1 +ibase.max_persistent = -1 +ibase.max_links = -1 +ibase.timestampformat = "%Y-%m-%d %H:%M:%S" +ibase.dateformat = "%Y-%m-%d" +ibase.timeformat = "%H:%M:%S" + +[MySQL] +mysql.allow_local_infile = On +mysql.allow_persistent = On +mysql.cache_size = 2000 +mysql.max_persistent = -1 +mysql.max_links = -1 +mysql.default_port = +mysql.default_socket = +mysql.default_host = +mysql.default_user = +mysql.default_password = +mysql.connect_timeout = 60 +mysql.trace_mode = Off + +[MySQLi] +mysqli.max_persistent = -1 +mysqli.allow_persistent = On +mysqli.max_links = -1 +mysqli.cache_size = 2000 +mysqli.default_port = 3306 +mysqli.default_socket = +mysqli.default_host = +mysqli.default_user = +mysqli.default_pw = +mysqli.reconnect = Off + +[mysqlnd] +mysqlnd.collect_statistics = On +mysqlnd.collect_memory_statistics = Off + +[OCI8] + +[PostgreSQL] +pgsql.allow_persistent = On +pgsql.auto_reset_persistent = Off +pgsql.max_persistent = -1 +pgsql.max_links = -1 +pgsql.ignore_notice = 0 +pgsql.log_notice = 0 + +[Sybase-CT] +sybct.allow_persistent = On +sybct.max_persistent = -1 +sybct.max_links = -1 +sybct.min_server_severity = 10 +sybct.min_client_severity = 10 + +[bcmath] +bcmath.scale = 0 + +[browscap] + +[Session] +session.save_handler = files +session.use_strict_mode = 0 +session.use_cookies = 1 +session.use_only_cookies = 1 +session.name = PHPSESSID +session.auto_start = 0 +session.cookie_lifetime = 0 +session.cookie_path = / +session.cookie_domain = +session.cookie_httponly = +session.serialize_handler = php +session.gc_probability = 0 +session.gc_divisor = 1000 +session.gc_maxlifetime = 1440 +session.referer_check = +session.cache_limiter = nocache +session.cache_expire = 180 +session.use_trans_sid = 0 +session.hash_function = 0 +session.hash_bits_per_character = 5 +url_rewriter.tags = "a=href,area=href,frame=src,input=src,form=fakeentry" + +[MSSQL] +mssql.allow_persistent = On +mssql.max_persistent = -1 +mssql.max_links = -1 +mssql.min_error_severity = 10 +mssql.min_message_severity = 10 +mssql.compatibility_mode = Off +mssql.secure_connection = Off + +[Assertion] + +[COM] + +[mbstring] + +[gd] + +[exif] + +[Tidy] +tidy.clean_output = Off + +[soap] +soap.wsdl_cache_enabled=1 +soap.wsdl_cache_dir="/tmp" +soap.wsdl_cache_ttl=86400 +soap.wsdl_cache_limit = 5 + +[sysvshm] + +[ldap] +ldap.max_links = -1 + +[mcrypt] + +[dba] + +[opcache] + +[curl] + +[openssl] + +; End: diff --git a/htadmin/supervisord.conf b/htadmin/supervisord.conf new file mode 100644 index 000000000..5d0196c94 --- /dev/null +++ b/htadmin/supervisord.conf @@ -0,0 +1,32 @@ +; Copyright (c) 2019 Battelle Energy Alliance, LLC. All rights reserved. + +[unix_http_server] +file=/var/run/supervisor.sock ; (the path to the socket file) +chmod=0700 + +[supervisord] +nodaemon=true +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +childlogdir=/var/log/supervisor + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[program:php] +command=php-fpm7.3 -F -R -g /var/run/php-fpm.pid +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:nginx] +command=/bin/bash -c "sleep 10 && nginx -g \"daemon off;\"" +startsecs=15 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true diff --git a/iso-build/build.sh b/iso-build/build.sh index 968401348..ece619088 100755 --- a/iso-build/build.sh +++ b/iso-build/build.sh @@ -85,6 +85,7 @@ if [ -d "$WORKDIR" ]; then MALCOLM_DEST_DIR="$WORKDIR/work/$IMAGE_NAME-Live-Build/config/includes.chroot/etc/skel/Malcolm" mkdir -p "$MALCOLM_DEST_DIR" mkdir -p "$MALCOLM_DEST_DIR/nginx/certs/" + mkdir -p "$MALCOLM_DEST_DIR/htadmin/" mkdir -p "$MALCOLM_DEST_DIR/logstash/certs/" mkdir -p "$MALCOLM_DEST_DIR/filebeat/certs/" mkdir -p "$MALCOLM_DEST_DIR/elasticsearch/nodes/" @@ -150,7 +151,7 @@ if [ -d "$WORKDIR" ]; then --linux-packages "linux-image-$(uname -r | sed 's/-amd64$//')" \ --architectures amd64 \ --binary-images iso-hybrid \ - --bootloaders "syslinux,grub-efi" \ + --bootloaders "grub-efi" \ --chroot-filesystem squashfs \ --backports false \ --security true \ @@ -164,7 +165,7 @@ if [ -d "$WORKDIR" ]; then --mirror-bootstrap http://ftp.us.debian.org/debian/ \ --mirror-binary http://httpredir.debian.org/debian/ \ --debootstrap-options "--include=apt-transport-https,gnupg,ca-certificates,openssl" \ - --apt-options "--force-yes --yes" + --apt-options "--allow-downgrades --allow-remove-essential --allow-change-held-packages --yes" lb build 2>&1 | tee "$WORKDIR/output/$IMAGE_NAME-$IMAGE_VERSION-build.log" if [ -f "$IMAGE_NAME-amd64.hybrid.iso" ]; then diff --git a/iso-build/config/includes.chroot/etc/skel/.config/lxpanel/LXDE/panels/malcolm b/iso-build/config/includes.chroot/etc/skel/.config/lxpanel/LXDE/panels/malcolm new file mode 100644 index 000000000..1f380376f --- /dev/null +++ b/iso-build/config/includes.chroot/etc/skel/.config/lxpanel/LXDE/panels/malcolm @@ -0,0 +1,145 @@ +# lxpanel config file. Manually editing is not recommended. +# Use preference dialog in lxpanel to adjust config when you can. + +Global { + edge=top + monitor=0 + height=32 + align=left + widthtype=percent + width=100 + transparent=0 + background=0 + autohide=0 + heightwhenhidden=4 + tintcolor=#a0a0a0 + alpha=255 + setpartialstrut=1 + iconsize=24 +} +Plugin { + type=menu + Config { + system { + } + separator { + } + item { + command=run + } + separator { + } + item { + command=logout + image=gnome-logout + } + image=/usr/share/icons/gnome/32x32/places/start-here.png + } +} +Plugin { + type=separator + Config { + } +} +Plugin { + type=launchtaskbar + Config { + Button { + id=terminator.desktop + } + Button { + id=chromium.desktop + } + Button { + id=malcolm-readme.desktop + } + Button { + id=malcolm-moloch.desktop + } + Button { + id=malcolm-kibana.desktop + } + Button { + id=malcolm-upload.desktop + } + Button { + id=malcolm-users.desktop + } + Button { + id=malcolm-cyberchef.desktop + } + Button { + id=malcolm-start.desktop + } + Button { + id=malcolm-restart.desktop + } + Button { + id=malcolm-logs.desktop + } + Button { + id=malcolm-stop.desktop + } + IconsOnly=0 + FlatButton=0 + UseMouseWheel=0 + GroupedTasks=1 + DisableUpscale=0 + UseSmallerIcons=-1 + spacing=1 + } +} +Plugin { + type=space + Config { + } + expand=1 +} +Plugin { + type=separator + Config { + } +} +Plugin { + type=pager + Config { + } +} +Plugin { + type=separator + Config { + } +} +Plugin { + type=dclock + Config { + ClockFmt=%R + TooltipFmt=%A %x + BoldFont=0 + IconOnly=0 + CenterText=0 + } +} +Plugin { + type=separator + Config { + } +} +Plugin { + type=tray + Config { + } +} +Plugin { + type=separator + Config { + } +} +Plugin { + type=launchbar + Config { + Button { + id=lxde-logout.desktop + } + } +} diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/cyberchef.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/cyberchef.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-cyberchef.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/kibana.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-kibana.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/kibana.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-kibana.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/logs.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-logs.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/logs.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-logs.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/moloch.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-moloch.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/moloch.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-moloch.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/readme.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-readme.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/readme.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-readme.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/restart.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-restart.desktop old mode 100755 new mode 100644 similarity index 63% rename from iso-build/config/includes.chroot/etc/skel/Desktop/restart.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-restart.desktop index ce6d7902e..52974d132 --- a/iso-build/config/includes.chroot/etc/skel/Desktop/restart.desktop +++ b/iso-build/config/includes.chroot/usr/share/applications/malcolm-restart.desktop @@ -1,7 +1,7 @@ #!/usr/bin/env xdg-open [Desktop Entry] Name=Restart Malcolm -Exec=lxterminal --command=/bin/bash\ -l\ -c\ ~//Malcolm/scripts/restart.sh +Exec=lxterminal --command=/bin/bash\ -l\ -c\ ~/Malcolm/scripts/restart.sh Comment=Restart Malcolm Terminal=false Type=Application diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/start.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-start.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/start.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-start.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/stop.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-stop.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/stop.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-stop.desktop diff --git a/iso-build/config/includes.chroot/etc/skel/Desktop/upload.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-upload.desktop old mode 100755 new mode 100644 similarity index 100% rename from iso-build/config/includes.chroot/etc/skel/Desktop/upload.desktop rename to iso-build/config/includes.chroot/usr/share/applications/malcolm-upload.desktop diff --git a/iso-build/config/includes.chroot/usr/share/applications/malcolm-users.desktop b/iso-build/config/includes.chroot/usr/share/applications/malcolm-users.desktop new file mode 100644 index 000000000..95b223b97 --- /dev/null +++ b/iso-build/config/includes.chroot/usr/share/applications/malcolm-users.desktop @@ -0,0 +1,11 @@ +[Desktop Entry] +Version=1.0 +Name=Malcolm - User Management +Exec=/usr/bin/chromium https://localhost:488/ +Terminal=false +X-MultipleArgs=false +Type=Application +Icon=config-users.png +Categories=Network; +StartupWMClass=chromium +StartupNotify=true diff --git a/iso-build/config/includes.chroot/usr/share/applications/users.desktop b/iso-build/config/includes.chroot/usr/share/applications/users.desktop new file mode 100755 index 000000000..95b223b97 --- /dev/null +++ b/iso-build/config/includes.chroot/usr/share/applications/users.desktop @@ -0,0 +1,11 @@ +[Desktop Entry] +Version=1.0 +Name=Malcolm - User Management +Exec=/usr/bin/chromium https://localhost:488/ +Terminal=false +X-MultipleArgs=false +Type=Application +Icon=config-users.png +Categories=Network; +StartupWMClass=chromium +StartupNotify=true diff --git a/iso-build/config/package-lists/system.list.chroot b/iso-build/config/package-lists/system.list.chroot index d732c67ec..2363ec308 100644 --- a/iso-build/config/package-lists/system.list.chroot +++ b/iso-build/config/package-lists/system.list.chroot @@ -9,6 +9,7 @@ audispd-plugins auditd bash-completion bc +bcrypt bridge-utils btrfs-progs busybox diff --git a/moloch/etc/config.ini b/moloch/etc/config.ini index 5f4371c4f..2ca9098cf 100644 --- a/moloch/etc/config.ini +++ b/moloch/etc/config.ini @@ -28,7 +28,11 @@ rirFile=/data/moloch/etc/ipv4-address-space.csv ouiFile=/data/moloch/etc/oui.txt dropUser=nobody dropGroup=daemon -userNameHeader=moloch_user +# implicit auto-creation of users for Moloch (see https://github.com/aol/moloch/pull/1120) +# The userAutoCreateTmpl should more or less match what's in /etc/user_settings.json +# which is what's used when creating the default admin user. +userNameHeader=http_auth_http_user +userAutoCreateTmpl={"userId": "${this.http_auth_http_user}", "userName": "${this.http_auth_http_user}", "enabled": true, "createEnabled": false, "webEnabled": true, "headerAuthEnabled": true, "emailSearch": true, "removeEnabled": false, "packetSearch": true, "hideStats": false, "hideFiles": false, "hidePcap": false, "disablePcapDownload": false, "settings": { "timezone": "local", "detailFormat": "last", "showTimestamps": "last", "sortColumn": "start", "sortDirection": "asc", "spiGraph": "protocol", "connSrcField": "srcIp", "connDstField": "dstIp", "numPackets": "last", "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF", "manualQuery": false }, "views": { "Public IP Addresses": { "expression": "(country.dst == EXISTS!) || (country.src == EXISTS!) || (ip.dst == EXISTS! && ip.dst != 0.0.0.0 && ip.dst != 127.0.0.1 && ip.dst != 224.0.0.0/4 && ip.dst != 255.255.255.255 && ip.dst != 10.0.0.0/8 && ip.dst != 172.16.0.0/12 && ip.dst != 192.168.0.0/16 && ip.dst != :: && ip.dst != ::1 && ip.dst != ff00::/8 && ip.dst != fe80::/10 && ip.dst != fc00::/7 && ip.dst != fd00::/8) || (ip.src == EXISTS! && ip.src != 0.0.0.0 && ip.src != 127.0.0.1 && ip.src != 224.0.0.0/4 && ip.src != 255.255.255.255 && ip.src != 10.0.0.0/8 && ip.src != 172.16.0.0/12 && ip.src != 192.168.0.0/16 && ip.src != :: && ip.src != ::1 && ip.src != ff00::/8 && ip.src != fe80::/10 && ip.src != fc00::/7 && ip.src != fd00::/8)" }, "PCAP Files": { "expression": "zeek.logType != EXISTS!" }, "Zeek Logs": { "expression": "zeek.logType == EXISTS!" }, "Zeek conn.log": { "expression": "zeek.logType == conn" }, "Zeek Exclude conn.log": { "expression": "zeek.logType == EXISTS! && zeek.logType != conn" } }, "tableStates": { "sessionsNew": { "order": [ [ "firstPacket", "asc" ] ], "visibleHeaders": [ "protocol", "zeek.logType", "firstPacket", "lastPacket", "src", "srcPort", "dst", "dstPort", "totPackets", "dbby", "tags", "info" ] } } } parseSMTP=true parseSMB=true parseQSValue=false diff --git a/moloch/patch/moloch_enable_implicit_creation_of_users_pull_1120_4d3f69a.patch b/moloch/patch/moloch_enable_implicit_creation_of_users_pull_1120_4d3f69a.patch new file mode 100644 index 000000000..8a08b6c9c --- /dev/null +++ b/moloch/patch/moloch_enable_implicit_creation_of_users_pull_1120_4d3f69a.patch @@ -0,0 +1,89 @@ +From 4d3f69aff152d7f44bbed677ea0473e94db3c254 Mon Sep 17 00:00:00 2001 +From: Scott Schenkein +Date: Mon, 8 Jul 2019 16:01:37 -0400 +Subject: [PATCH] Merge of header authorization and implicit user creation + +--- + viewer/viewer.js | 44 +++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 43 insertions(+), 1 deletion(-) + +diff --git a/viewer/viewer.js b/viewer/viewer.js +index 54c11a52..955d0aca 100644 +--- a/viewer/viewer.js ++++ b/viewer/viewer.js +@@ -65,6 +65,9 @@ var internals = { + elasticBase: Config.get("elasticsearch", "http://localhost:9200").split(","), + esQueryTimeout: Config.get("elasticsearchTimeout", 300) + 's', + userNameHeader: Config.get("userNameHeader"), ++ requiredAuthHeader: Config.get("requiredAuthHeader"), ++ requiredAuthHeaderVal: Config.get("requiredAuthHeaderVal"), ++ userAutoCreateTmpl: Config.get("userAutoCreateTmpl"), + httpAgent: new http.Agent({keepAlive: true, keepAliveMsecs:5000, maxSockets: 40}), + httpsAgent: new https.Agent({keepAlive: true, keepAliveMsecs:5000, maxSockets: 40, rejectUnauthorized: !Config.insecure}), + previousNodesStats: [], +@@ -280,8 +283,26 @@ if (Config.get("passwordSecret")) { + // Header auth + if (internals.userNameHeader !== undefined) { + if (req.headers[internals.userNameHeader] !== undefined) { ++ // Check if we require a certain header+value to be present ++ // as in the case of an apache plugin that sends AD groups ++ if (internals.requiredAuthHeader !== undefined && internals.requiredAuthHeaderVal !== undefined) { ++ var authHeader = req.headers[internals.requiredAuthHeader]; ++ if (authHeader === undefined) { ++ return res.send("Missing authorization header"); ++ } ++ var authorized = false; ++ authHeader.split(",").forEach(headerVal => { ++ if (headerVal.trim() == internals.requiredAuthHeaderVal) { ++ authorized = true; ++ } ++ }); ++ if (!authorized) { ++ return res.send("Not authorized"); ++ } ++ } + var userName = req.headers[internals.userNameHeader]; +- Db.getUserCache(userName, function(err, suser) { ++ ++ function ucb(err, suser) { + if (err) {return res.send("ERROR - getUser - user: " + userName + " err:" + err);} + if (!suser || !suser.found) {return res.send(userName + " doesn't exist");} + if (!suser._source.enabled) {return res.send(userName + " not enabled");} +@@ -290,6 +311,26 @@ if (Config.get("passwordSecret")) { + userCleanup(suser._source); + req.user = suser._source; + return next(); ++ }; ++ ++ Db.getUserCache(userName, function(err, suser) { ++ if (internals.userAutoCreateTmpl === undefined) { ++ return ucb(err, suser); ++ } else if ((err && err.toString().includes("Not Found")) || ++ (!suser || !suser.found)) { // Try dynamic creation ++ var nuser = JSON.parse(new Function("return `" + ++ internals.userAutoCreateTmpl + "`;").call(req.headers)); ++ Db.setUser(userName, nuser, (err, info) => { ++ if (err) { ++ console.log("Elastic search error adding user: (" + userName + "):(" + JSON.stringify(nuser) + "):" + err); ++ } else { ++ console.log("Added user:" + userName + ":" + JSON.stringify(nuser)); ++ } ++ return Db.getUserCache(userName, ucb); ++ }); ++ } else { ++ return ucb(err, suser); ++ } + }); + return; + } else if (Config.debug) { +@@ -297,6 +338,7 @@ if (Config.get("passwordSecret")) { + } + } + ++ + // Browser auth + req.url = req.url.replace("/", Config.basePath()); + passport.authenticate('digest', {session: false})(req, res, function (err) { +-- +2.20.1 + diff --git a/moloch/scripts/bs4_remove_div.py b/moloch/scripts/bs4_remove_div.py new file mode 100755 index 000000000..aee5595dd --- /dev/null +++ b/moloch/scripts/bs4_remove_div.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import print_function + +from bs4 import BeautifulSoup +import argparse +import os +import sys + +################################################################################################### +debug = False +PY3 = (sys.version_info.major >= 3) +scriptName = os.path.basename(__file__) +scriptPath = os.path.dirname(os.path.realpath(__file__)) +origPath = os.getcwd() + +################################################################################################### +if not PY3: + if hasattr(__builtins__, 'raw_input'): input = raw_input + +try: + FileNotFoundError +except NameError: + FileNotFoundError = IOError + +################################################################################################### +# print to stderr +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +################################################################################################### +# convenient boolean argument parsing +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +################################################################################################### +# main +def main(): + global debug + + parser = argparse.ArgumentParser(description=scriptName, add_help=False, usage='{} '.format(scriptName)) + parser.add_argument('-v', '--verbose', dest='debug', type=str2bool, nargs='?', const=True, default=False, help="Verbose output") + parser.add_argument('-i', '--input', required=True, metavar='', type=str, help='Input file') + parser.add_argument('-o', '--output', required=True, metavar='', type=str, help='Output file') + parser.add_argument('-c', '--div-class', required=True, dest='divClass', metavar='', type=str, default='', help='div class to remove') + parser.add_argument('-p', '--parser', required=False, dest='parser', metavar='', type=str, default='html.parser', help='BeautifulSoup parser') + parser.add_argument('-e', '--encoding', required=False, dest='encoding', metavar='', type=str, default='utf-8', help='Encoding for output file') + try: + parser.error = parser.exit + args = parser.parse_args() + except SystemExit: + parser.print_help() + exit(2) + + debug = args.debug + if debug: + eprint(os.path.join(scriptPath, scriptName)) + eprint("Arguments: {}".format(sys.argv[1:])) + eprint("Arguments: {}".format(args)) + else: + sys.tracebacklimit = 0 + + soup = BeautifulSoup(open(args.input), args.parser) + for div in soup.find_all("div", { 'class' : args.divClass }): + div.decompose() + + with open(args.output, 'wb') as f: + f.write(soup.prettify(args.encoding)) + +if __name__ == '__main__': + main() diff --git a/nginx/nginx.conf b/nginx/nginx.conf index ed0df2937..5c4f9031a 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -30,6 +30,10 @@ http { server upload:80; } + upstream docker-htadmin { + server htadmin:80; + } + upstream docker-kibana { server kibana:5601; } @@ -50,6 +54,20 @@ http { server logstash:9600; } + # htadmin (htpasswd/user management) + server { + listen 488; + ssl on; + ssl_certificate /etc/nginx/certs/cert.pem; + ssl_certificate_key /etc/nginx/certs/key.pem; + + location / { + proxy_pass http://docker-htadmin; + proxy_redirect off; + proxy_set_header Host htadmin.malcolm.local; + } + } + # Moloch interface server { listen 443; @@ -100,7 +118,7 @@ http { proxy_pass http://docker-moloch; proxy_redirect off; proxy_set_header Host moloch.malcolm.local; - proxy_set_header MOLOCH_USER $remote_user; + proxy_set_header http_auth_http_user $remote_user; proxy_set_header Authorization ""; } } diff --git a/scripts/auth_setup.sh b/scripts/auth_setup.sh index 6413674f6..2f9dd7b14 100755 --- a/scripts/auth_setup.sh +++ b/scripts/auth_setup.sh @@ -29,7 +29,7 @@ USERNAME="" PASSWORD="" PASSWORD_CONFIRM="" -read -p "Username: " USERNAME +read -p "Administrator username: " USERNAME while true; do read -s -p "${USERNAME} password: " PASSWORD echo @@ -40,16 +40,54 @@ while true; do done PASSWORD_ENCRYPTED="$(echo $PASSWORD | openssl passwd -1 -stdin)" +# get previous admin username to remove from htpasswd file if it's changed +unset USERNAME_PREVIOUS +[[ -r auth.env ]] && source auth.env && USERNAME_PREVIOUS="$MALCOLM_USERNAME" + cat < auth.env -# username and encrypted password for nginx reverse proxy (and upload server's SFTP access) +# Malcolm Administrator username and encrypted password for nginx reverse proxy (and upload server's SFTP access) MALCOLM_USERNAME=$USERNAME MALCOLM_PASSWORD=$PASSWORD_ENCRYPTED EOF - chmod 600 ./auth.env pushd ./nginx/ >/dev/null 2>&1 -htpasswd -bc ./htpasswd "$USERNAME" "$PASSWORD" >/dev/null 2>&1 +# create or update the htpasswd file +[[ ! -f ./htpasswd ]] && HTPASSWD_CREATE_FLAG="-c" || HTPASSWD_CREATE_FLAG="" +htpasswd -b $HTPASSWD_CREATE_FLAG -B ./htpasswd "$USERNAME" "$PASSWORD" >/dev/null 2>&1 +# grab the hashed version of the password to also store in the htadmin/config.ini file +PASSWORD_HTPASSWD_HASHED="$(grep "^$USERNAME:" ./htpasswd | head -n 1 | cut -d: -f2)" +# if the admininstrator username has changed, remove the previous administrator username from htpasswd +[[ -n "$USERNAME_PREVIOUS" ]] && [ "$USERNAME" != "$USERNAME_PREVIOUS" ] && sed -i "/^$USERNAME_PREVIOUS:/d" ./htpasswd + +popd >/dev/null 2>&1 + +pushd ./htadmin/ >/dev/null 2>&1 +cat < config.ini +; HTAdmin config file. + +[application] +; Change this to customize your title: +app_title = Malcolm User Management + +; htpasswd file +secure_path = ./config/htpasswd +; metadata file +metadata_path = ./config/metadata + +; administrator user/password (htpasswd -b -c -B ...) +admin_user = $USERNAME +admin_pwd_hash = $PASSWORD_HTPASSWD_HASHED + +; SMTP server information for password reset: +mail_from = admin@example.com +mail_from_name = Administrator +mail_user = admin@example.com +mail_pwd = xxxx +mail_server = mail.example.com + +EOF +touch metadata popd >/dev/null 2>&1 unset CONFIRMATION diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index a549d3d34..a35d797f4 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -62,6 +62,7 @@ if mkdir "$DESTDIR"; then trap "cleanup" EXIT mkdir $VERBOSE -p "$DESTDIR/nginx/certs/" + mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" mkdir $VERBOSE -p "$DESTDIR/filebeat/certs/" mkdir $VERBOSE -p "$DESTDIR/elasticsearch/nodes/" @@ -98,7 +99,7 @@ if mkdir "$DESTDIR"; then cp $VERBOSE ./elastalert/rules/* "$DESTDIR/elastalert/rules/" 2>/dev/null || true cp $VERBOSE ./elastalert/sample-rules/* "$DESTDIR/elastalert/sample-rules/" 2>/dev/null || true pushd "$DESTDIR" >/dev/null 2>&1 - echo "You must set a username and password for Malcolm, and self-signed X.509 certificates will be generated" + echo "You must set an administrator username and password for Malcolm, and self-signed X.509 certificates will be generated" ./scripts/auth_setup.sh rm -rf logstash/certs/ca.key pushd .. >/dev/null 2>&1 @@ -138,10 +139,11 @@ if mkdir "$DESTDIR"; then echo " - auth_setup.sh (change authentication-related settings)" | tee -a "$README" echo "" | tee -a "$README" echo "A minute or so after starting Malcolm, the following services will be accessible:" | tee -a "$README" - echo " - Moloch: https://localhost/" | tee -a "$README" + echo " - Moloch: https://localhost:443/" | tee -a "$README" echo " - Kibana: https://localhost:5601/" | tee -a "$README" echo " - PCAP Upload (web): https://localhost:8443/" | tee -a "$README" echo " - PCAP Upload (sftp): sftp://USERNAME@127.0.0.1:8022/files/" | tee -a "$README" + echo " - Account management: https://localhost:488/" | tee -a "$README" popd >/dev/null 2>&1 popd >/dev/null 2>&1 popd >/dev/null 2>&1 diff --git a/scripts/start.sh b/scripts/start.sh index 34a8373d8..983f89489 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -30,19 +30,21 @@ pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 # if we are in an interactive shell and we're missing any of the auth files, prompt to create them now # ( another way to check this: [[ "${-}" =~ 'i' ]] ) if [[ -t 1 ]] && \ - ( [[ ! -f ./nginx/htpasswd ]] || [[ ! -f ./nginx/certs/cert.pem ]] || [[ ! -f ./nginx/certs/key.pem ]] || [[ ! -r ./auth.env ]] ) + ( [[ ! -f ./nginx/htpasswd ]] || [[ ! -f ./htadmin/config.ini ]] || [[ ! -f ./nginx/certs/cert.pem ]] || [[ ! -f ./nginx/certs/key.pem ]] || [[ ! -r ./auth.env ]] ) then - echo "Malcolm authentication files are missing, running ./scripts/auth_setup.sh..." + echo "Malcolm administrator account authentication files are missing, running ./scripts/auth_setup.sh..." ./scripts/auth_setup.sh fi # still missing? sorry charlie -if [[ ! -f ./nginx/htpasswd ]] || [[ ! -f ./nginx/certs/cert.pem ]] || [[ ! -f ./nginx/certs/key.pem ]] || [[ ! -r ./auth.env ]]; then - echo "Malcolm authentication files are missing, please run ./scripts/auth_setup.sh to generate them" +if [[ ! -f ./nginx/htpasswd ]] || [[ ! -f ./htadmin/config.ini ]] || [[ ! -f ./nginx/certs/cert.pem ]] || [[ ! -f ./nginx/certs/key.pem ]] || [[ ! -r ./auth.env ]]; then + echo "Malcolm administrator account authentication files are missing, please run ./scripts/auth_setup.sh to generate them" exit 1 fi -# make sure a read permission is set correctly for the nginx worker process -chmod 644 ./nginx/htpasswd >/dev/null 2>&1 +[[ -f ./htadmin/metadata ]] || touch ./htadmin/metadata + +# make sure a read permission is set correctly for the nginx worker processes +chmod 644 ./nginx/htpasswd ./htadmin/config.ini ./htadmin/metadata >/dev/null 2>&1 # make sure some directories exist before we start mkdir -p ./elasticsearch/ @@ -66,10 +68,11 @@ if $DOCKER_COMPOSE_COMMAND up --detach ; then echo "In a few minutes, Malcolm services will be accessible via the following URLs:" echo "------------------------------------------------------------------------------" - echo " - Moloch: https://localhost/" + echo " - Moloch: https://localhost:443/" echo " - Kibana: https://localhost:5601/" echo " - PCAP Upload (web): https://localhost:8443/" echo " - PCAP Upload (sftp): sftp://username@127.0.0.1:8022/files/" + echo " - Account management: https://localhost:488/" echo "" $SCRIPT_PATH/logs.sh "$CONFIG_FILE"