multiple user accounts and account management (#39)

* bump version for next point release

* increase length of self-signed cert

* Prompt during installation for whether or not to disable IPv6

* In the Kibana Software dashboard pie chart, don't exclude software for which a version is unknown

* Allow Malcolm to be restarted automatically upon docker daemon restart

* sync malcolm and iso installer

* working on using htadmin as a user management frontend

* working more on account management

* working on multiple users

* tweak nginx according to arkime/arkime#1120

* more work on multi-user auto creation with moloch

* more work on multi-user auto creation with moloch

* more work on multi-user auto creation with moloch

* more work on multi-user auto creation with moloch

* bump version, udpate README for account management

* Improved comments for zeeklogs wise data source

* added user desktop file for malcolm iso

* create missing htadmin directory in iso build

* relocate icons for malcolm from desktop

* BIOS ISO is kind of busted at the moment anyway as far as preseeding goes, so for now just install the EFI bootloader.

* fix read permissions for group/world on htpasswd file for nginx processes

* A couple fixes for the iso: 1) disable some callback features in chromium via policy, 2) let wicd manage networking

* When running auth_setup.sh to reset the administrator password, don't blow away the other passwords in the database

* working on using htadmin as a user management frontend

* working on multiple users

* tweak nginx according to arkime/arkime#1120

* more work on multi-user auto creation with moloch

* more work on multi-user auto creation with moloch

* added user desktop file for malcolm iso

.gitignore

@@ -10,6 +10,8 @@
 /filebeat/certs
 /nginx/certs
 /nginx/htpasswd
+/htadmin/config.ini
+/htadmin/metadata
 
 # development
 .vagrant

Dockerfiles/file-monitor.Dockerfile

@@ -58,8 +58,8 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       python3-requests && \
     pip3 install clamd namedlist supervisor && \
     mkdir -p /var/log/supervisor && \
-    apt-get -y -q --force-yes --purge remove python3-dev build-essential && \
-      apt-get -y -q --force-yes autoremove && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove python3-dev build-essential && \
+      apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
       apt-get clean && \
       rm -rf /var/lib/apt/lists/* && \
     wget -O /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \

Dockerfiles/file-upload.Dockerfile

@@ -10,7 +10,7 @@ ENV JQUERY_FILE_UPLOAD_VERSION v9.19.1
 ADD https://github.com/blueimp/jQuery-File-Upload/archive/${JQUERY_FILE_UPLOAD_VERSION}.tar.gz /jQuery-File-Upload.tar.gz
 
 RUN apt-get update && \
-    apt-get -y -q --force-yes install --no-install-recommends npm node-encoding git ca-certificates wget && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends npm node-encoding git ca-certificates wget && \
     npm install -g bower && \
     mkdir /jQuery-File-Upload && \
   	tar --strip-components=1 -C /jQuery-File-Upload -xzf /jQuery-File-Upload.tar.gz && \
@@ -30,9 +30,8 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
 RUN apt-get update && \
-    apt-get -y -q --force-yes install --no-install-recommends \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends \
       wget \
-      apt-transport-https \
       ca-certificates \
       openssh-server \
       supervisor \
@@ -42,7 +41,8 @@ RUN apt-get update && \
       php7.3-fpm \
       php7.3-apcu \
       nginx-light && \
-    apt-get clean -y -q
+    apt-get clean -y -q && \
+    rm -rf /var/lib/apt/lists/*
 
 ADD file-upload/supervisord.conf /supervisord.conf
 ADD file-upload/jquery-file-upload/index.html /var/www/upload/index.html

Dockerfiles/htadmin.Dockerfile

@@ -0,0 +1,66 @@
+FROM debian:buster-slim
+# Copyright (c) 2019 Battelle Energy Alliance, LLC.  All rights reserved.
+LABEL maintainer="[email protected]"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM xterm
+
+ARG PHP_VERSION=7.3
+ARG MCRYPT_VERSION=1.0.2
+ARG BOOTSTRAP_VERSION=3.3.6
+
+ENV PHP_VERSION $PHP_VERSION
+ENV MCRYPT_VERSION $MCRYPT_VERSION
+ENV BOOTSTRAP_VERSION $BOOTSTRAP_VERSION
+
+RUN apt-get update && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends install \
+      bcrypt \
+      ca-certificates \
+      curl \
+      git \
+      libmcrypt-dev \
+      libmcrypt4 \
+      make \
+      mcrypt \
+      nginx-light \
+      php-dev \
+      php-pear \
+      php$PHP_VERSION-apcu \
+      php$PHP_VERSION-cli \
+      php$PHP_VERSION-curl \
+      php$PHP_VERSION-fpm \
+      php$PHP_VERSION-gd \
+      procps \
+      supervisor && \
+    ( yes '' | pecl channel-update pecl.php.net ) && \
+    ( yes '' | pecl install mcrypt-$MCRYPT_VERSION ) && \
+    ln -s -r /usr/lib/php/20??????/*.so /usr/lib/php/$PHP_VERSION/ && \
+    mkdir -p /run/php && \
+    git clone --depth 1 https://github.com/mmguero/htadmin /tmp/htadmin && \
+      mv /tmp/htadmin/sites/html/htadmin /var/www/htadmin && \
+      cd /var/www/htadmin && \
+      ( grep -rhoPi "(src|href)=['\"]https?://.+?['\"]" ./includes/* | sed "s/^[a-zA-Z]*=['\"]*//" | sed "s/['\"]$//" | xargs -r -l curl -s -S -L -J -O ) && \
+      sed -i "s@http[^'\"]*/@@gI" ./includes/* && \
+      mkdir fonts && cd fonts && \
+      curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.ttf" && \
+      curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff" && \
+      curl -s -S -L -J -O "https://maxcdn.bootstrapcdn.com/bootstrap/$BOOTSTRAP_VERSION/fonts/glyphicons-halflings-regular.woff2" && \
+    cd /tmp && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --purge remove \
+      git make libmcrypt-dev php-pear php-dev && \
+    apt-get autoremove -y -q && \
+    apt-get clean -y -q && \
+    usermod --non-unique --uid 1000 www-data && \
+      groupmod --non-unique --gid 1000 www-data && \
+      chown -R www-data:www-data /var/www && \
+    rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /var/tmp/* /var/www/html
+
+ADD docs/images/favicon/favicon.ico /var/www/htadmin/
+ADD htadmin/supervisord.conf /supervisord.conf
+ADD htadmin/php/php.ini /etc/php/$PHP_VERSION/fpm/php.ini
+ADD htadmin/nginx/sites-available/default /etc/nginx/sites-available/default
+
+EXPOSE 80
+
+CMD ["/usr/bin/supervisord", "-c", "/supervisord.conf", "-u", "root", "-n"]

Dockerfiles/moloch.Dockerfile

@@ -11,6 +11,7 @@ ENV ZEEK_VERSION "2.6.2"
 ENV ZEEK_DIR "/opt/bro"
 ENV CYBERCHEF_VERSION "8.30.1"
 
+ADD moloch/scripts/bs4_remove_div.py /data/
 ADD moloch/patch/* /data/patches/
 ADD README.md $MOLOCHDIR/doc/
 ADD doc.css $MOLOCHDIR/doc/
@@ -49,11 +50,13 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list
         patch \
         python-dev \
         python3-dev \
+        python3-pip \
         rename \
         sudo \
         swig \
         wget \
         zlib1g-dev && \
+  pip3 install --no-cache-dir beautifulsoup4 && \
   cd /data && \
   tar -xvf "bro.tar.gz" && \
     rm -f "bro.tar.gz" && \
@@ -101,6 +104,8 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list
     ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \
     ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \
     ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \
+    python3 /data/bs4_remove_div.py -i ./viewer/vueapp/src/components/users/Users.vue -o ./viewer/vueapp/src/components/users/Users.new -c "new-user-form" && \
+    mv -vf ./viewer/vueapp/src/components/users/Users.new ./viewer/vueapp/src/components/users/Users.vue && \
     ./easybutton-build.sh --install && \
     npm cache clean --force && \
   apt-get clean && \
@@ -195,7 +200,7 @@ RUN sed -i "s/stretch main/stretch main contrib non-free/" /etc/apt/sources.list
       vim-tiny \
       wget \
       tar gzip unzip cpio bzip2 lzma xz-utils p7zip-full unrar zlib1g && \
-    pip3 install --no-cache-dir elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \
+    pip3 install --no-cache-dir beautifulsoup4 elasticsearch manuf geoip2 patool entrypoint2 pyunpack && \
     ln -sf $MOLOCHDIR/bin/npm /usr/local/bin/npm && \
       ln -sf $MOLOCHDIR/bin/node /usr/local/bin/node && \
       ln -sf $MOLOCHDIR/bin/npx /usr/local/bin/npx && \