Work in progress in converting to opensearch (see #54)

Dockerfiles/arkime.Dockerfile

@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV ARKIME_VERSION "2.7.1"
 ENV ARKIMEDIR "/data/moloch"
 ENV ARKIME_URL "https://codeload.github.com/arkime/arkime/tar.gz/v${ARKIME_VERSION}"
-ENV ARKIME_LOCALELASTICSEARCH no
+ENV ARKIME_LOCALOPENSEARCH no
 ENV ARKIME_INET yes
 
 ADD moloch/scripts/bs4_remove_div.py /data/
@@ -78,7 +78,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
     mv -vf ./viewer/vueapp/src/components/users/Users.new ./viewer/vueapp/src/components/users/Users.vue && \
     sed -i 's/v-if.*password.*"/v-if="false"/g' ./viewer/vueapp/src/components/settings/Settings.vue && \
     rm -rf ./viewer/vueapp/src/components/upload && \
-    sed -i "s/^\(ARKIME_LOCALELASTICSEARCH=\).*/\1"$ARKIME_LOCALELASTICSEARCH"/" ./release/Configure && \
+    sed -i "s/^\(ARKIME_LOCALOPENSEARCH=\).*/\1"$ARKIME_LOCALOPENSEARCH"/" ./release/Configure && \
     sed -i "s/^\(ARKIME_INET=\).*/\1"$ARKIME_INET"/" ./release/Configure && \
     ./easybutton-build.sh --install && \
     npm cache clean --force && \
@@ -106,8 +106,8 @@ ENV PUSER_PRIV_DROP true
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
-ARG ES_HOST=elasticsearch
-ARG ES_PORT=9200
+ARG ES_HOST=opensearch
+ARG OS_PORT=9200
 ARG MALCOLM_USERNAME=admin
 ARG ARKIME_INTERFACE=eth0
 ARG ARKIME_ANALYZE_PCAP_THREADS=1
@@ -124,8 +124,8 @@ ARG MAXMIND_GEOIP_DB_LICENSE_KEY=""
 
 # Declare envs vars for each arg
 ENV ES_HOST $ES_HOST
-ENV ES_PORT $ES_PORT
-ENV ARKIME_ELASTICSEARCH "http://"$ES_HOST":"$ES_PORT
+ENV OS_PORT $OS_PORT
+ENV ARKIME_OPENSEARCH "http://"$ES_HOST":"$OS_PORT
 ENV ARKIME_INTERFACE $ARKIME_INTERFACE
 ENV MALCOLM_USERNAME $MALCOLM_USERNAME
 # this needs to be present, but is unused as nginx is going to handle auth for us
@@ -186,7 +186,7 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD moloch/scripts /data/
 ADD shared/bin/pcap_moloch_and_zeek_processor.py /data/
 ADD shared/bin/pcap_utils.py /data/
-ADD shared/bin/elastic_search_status.sh /data/
+ADD shared/bin/opensearch_status.sh /data/
 ADD moloch/etc $ARKIMEDIR/etc/
 ADD moloch/wise/source.*.js $ARKIMEDIR/wiseService/
 ADD moloch/supervisord.conf /etc/supervisord.conf

Dockerfiles/file-monitor.Dockerfile

@@ -83,7 +83,7 @@ ENV SRC_BASE_DIR "/usr/local/src"
 ENV CLAMAV_RULES_DIR "/var/lib/clamav"
 ENV YARA_VERSION "4.1.1"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
-ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
+ENV YARA_RULOS_URL "https://github.com/Neo23x0/signature-base"
 ENV YARA_RULES_DIR "/yara-rules"
 ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
 ENV CAPA_VERSION "1.6.3"
@@ -156,7 +156,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
       make install && \
     rm -rf "${SRC_BASE_DIR}"/yara* && \
     cd /tmp && \
-      git clone --depth 1 --single-branch "${YARA_RULES_URL}" "${YARA_RULES_SRC_DIR}" && \
+      git clone --depth 1 --single-branch "${YARA_RULOS_URL}" "${YARA_RULES_SRC_DIR}" && \
       mkdir -p "${YARA_RULES_DIR}" && \
       ln -f -s -r "${YARA_RULES_SRC_DIR}"/yara/* "${YARA_RULES_SRC_DIR}"/vendor/yara/* "${YARA_RULES_DIR}"/ && \
     cd /tmp && \

Dockerfiles/filebeat.Dockerfile

@@ -62,7 +62,7 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
 ADD filebeat/filebeat-nginx.yml /usr/share/filebeat-nginx/filebeat-nginx.yml
 ADD filebeat/scripts /data/
-ADD shared/bin/elastic_search_status.sh /data/
+ADD shared/bin/opensearch_status.sh /data/
 ADD filebeat/supervisord.conf /etc/supervisord.conf
 RUN mkdir -p /usr/share/filebeat-nginx/data && \
     chown -R root:${PGROUP} /usr/share/filebeat-nginx && \

Dockerfiles/kibana-helper.Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.14
+  FROM alpine:3.14
 
 # Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -23,8 +23,8 @@ ENV TERM xterm
 ARG ARKIME_INDEX_PATTERN="sessions2-*"
 ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
 ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
-ARG CREATE_ES_ARKIME_SESSION_INDEX="true"
-ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
+ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
+ARG OPENSEARCH_URL="http://opensearch:9200"
 ARG ISM_SNAPSHOT_COMPRESSED=false
 ARG ISM_SNAPSHOT_REPO=logs
 ARG KIBANA_OFFLINE_REGION_MAPS_PORT="28991"
@@ -33,8 +33,8 @@ ARG KIBANA_URL="http://kibana:5601/kibana"
 ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
 ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
 ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
-ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
-ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
+ENV CREATE_OS_ARKIME_SESSION_INDEX $CREATE_OS_ARKIME_SESSION_INDEX
+ENV OPENSEARCH_URL $OPENSEARCH_URL
 ENV ISM_SNAPSHOT_COMPRESSED $ISM_SNAPSHOT_COMPRESSED
 ENV ISM_SNAPSHOT_REPO $ISM_SNAPSHOT_REPO
 ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
@@ -53,8 +53,8 @@ ADD kibana/scripts /data/
 ADD kibana/supervisord.conf /etc/supervisord.conf
 ADD kibana/zeek_template.json /data/zeek_template.json
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
-ADD shared/bin/elastic_search_status.sh /data/
-ADD shared/bin/elastic_index_size_prune.py /data/
+ADD shared/bin/opensearch_status.sh /data/
+ADD shared/bin/opensearch_index_size_prune.py /data/
 
 RUN apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq && \
     npm install -g http-server && \
@@ -72,7 +72,7 @@ RUN apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq &&
     chown -R ${PUSER}:${PGROUP} /opt/kibana/dashboards /opt/maps /data/init && \
     chmod 755 /data/*.sh /data/*.py /data/init && \
     chmod 400 /opt/maps/* && \
-    (echo -e "*/2 * * * * /data/kibana-create-moloch-sessions-index.sh\n0 10 * * * /data/kibana_index_refresh.py --template zeek_template\n*/20 * * * * /data/elastic_index_size_prune.py" > ${SUPERCRONIC_CRONTAB})
+    (echo -e "*/2 * * * * /data/kibana-create-moloch-sessions-index.sh\n0 10 * * * /data/kibana_index_refresh.py --template zeek_template\n*/20 * * * * /data/opensearch_index_size_prune.py" > ${SUPERCRONIC_CRONTAB})
 
 EXPOSE $KIBANA_OFFLINE_REGION_MAPS_PORT
 

Dockerfiles/kibana.Dockerfile

@@ -82,22 +82,22 @@ ENV PUSER_PRIV_DROP true
 
 ENV TERM xterm
 
-ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
-ARG CREATE_ES_ARKIME_SESSION_INDEX="true"
+ARG OPENSEARCH_URL="http://opensearch:9200"
+ARG CREATE_OS_ARKIME_SESSION_INDEX="true"
 ARG ARKIME_INDEX_PATTERN="sessions2-*"
 ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
 ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
 ARG KIBANA_DEFAULT_DASHBOARD="0ad3d7c2-3441-485e-9dfe-dbb22e84e576"
 
-ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
+ENV CREATE_OS_ARKIME_SESSION_INDEX $CREATE_OS_ARKIME_SESSION_INDEX
 ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
 ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
 ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
 ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
 ENV KIBANA_OFFLINE_REGION_MAPS $KIBANA_OFFLINE_REGION_MAPS
 ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
 ENV PATH="/data:${PATH}"
-ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
+ENV OPENSEARCH_URL $OPENSEARCH_URL
 ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
 
 USER root

Dockerfiles/logstash.Dockerfile

@@ -64,15 +64,15 @@ ENV TERM xterm
 
 ARG LOGSTASH_ENRICHMENT_PIPELINE=enrichment
 ARG LOGSTASH_PARSE_PIPELINE_ADDRESSES=zeek-parse
-ARG LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL=internal-es
-ARG LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL=external-es
-ARG LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES=internal-es,external-es
+ARG LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL=internal-es
+ARG LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL=external-es
+ARG LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES=internal-es,external-es
 
 ENV LOGSTASH_ENRICHMENT_PIPELINE $LOGSTASH_ENRICHMENT_PIPELINE
 ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES
-ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL
-ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL
-ENV LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES
+ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL
+ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL
+ENV LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES
 ENV JAVA_HOME=/usr/share/logstash/jdk
 
 USER root

Dockerfiles/elasticsearch.Dockerfile → Dockerfiles/opensearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM amazon/opendistro-for-elasticsearch:1.13.2
+FROM opensearchproject/opensearch:1.0.0
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -7,45 +7,44 @@ LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm'
 LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/master/README.md'
 LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm'
 LABEL org.opencontainers.image.vendor='Idaho National Laboratory'
-LABEL org.opencontainers.image.title='malcolmnetsec/elasticsearch-od'
-LABEL org.opencontainers.image.description='Malcolm container providing Elasticsearch (the Apache-licensed Open Distro variant)'
+LABEL org.opencontainers.image.title='malcolmnetsec/opensearch'
+LABEL org.opencontainers.image.description='Malcolm container providing OpenSearch'
 
 ARG DEFAULT_UID=1000
 ARG DEFAULT_GID=1000
 ENV DEFAULT_UID $DEFAULT_UID
 ENV DEFAULT_GID $DEFAULT_GID
 ENV PUID $DEFAULT_UID
-ENV PUSER "elasticsearch"
-ENV PGROUP "elasticsearch"
+ENV PUSER "opensearch"
+ENV PGROUP "opensearch"
 ENV PUSER_PRIV_DROP true
 
 ENV TERM xterm
 
 ARG GITHUB_OAUTH_TOKEN=""
 ARG DISABLE_INSTALL_DEMO_CONFIG=true
 ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG
-ENV JAVA_HOME=/usr/share/elasticsearch/jdk
+ENV JAVA_HOME=/usr/share/opensearch/jdk
+
+USER root
 
 # Malcolm manages authentication and encryption via NGINX reverse proxy
-# https://opendistro.github.io/for-elasticsearch-docs/docs/security/configuration/disable/
-# https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/#customize-the-docker-image
-# https://github.com/opendistro-for-elasticsearch/opendistro-build/issues/613
 RUN yum install -y openssl && \
-  /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
-  echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/elasticsearch/config/elasticsearch.yml && \
-  chown -R $PUSER:$PGROUP /usr/share/elasticsearch/config/elasticsearch.yml && \
-  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/plugins/opendistro-performance-analyzer/pa_config/supervisord.conf && \
-  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/elasticsearch/performance-analyzer-rca/pa_config/supervisord.conf && \
-  sed -i '/[^#].*\/usr\/share\/elasticsearch\/bin\/elasticsearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/local/bin/docker-entrypoint.sh
+  /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security && \
+  echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/opensearch/config/opensearch.yml && \
+  chown -R $PUSER:$PGROUP /usr/share/opensearch/config/opensearch.yml && \
+  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/opensearch/plugins/opensearch-performance-analyzer/pa_config/supervisord.conf && \
+  sed -i "s/user=1000\b/user=%(ENV_PUID)s/g" /usr/share/opensearch/performance-analyzer-rca/pa_config/supervisord.conf && \
+  sed -i '/[^#].*\$OPENSEARCH_HOME\/bin\/opensearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/share/opensearch/opensearch-docker-entrypoint.sh
+
+
 # just used for initial keystore creation
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
 
-USER root
-
 ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
 
-CMD ["/usr/local/bin/docker-entrypoint.sh"]
+CMD ["/usr/share/opensearch/opensearch-docker-entrypoint.sh"]
 
 # to be populated at build-time:
 ARG BUILD_DATE

Dockerfiles/pcap-monitor.Dockerfile

@@ -24,14 +24,14 @@ ENV PUSER_PRIV_DROP false
 ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
-ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
+ARG OPENSEARCH_URL="http://opensearch:9200"
 ARG PCAP_PATH=/pcap
 ARG PCAP_PIPELINE_DEBUG=false
 ARG PCAP_PIPELINE_DEBUG_EXTRA=false
 ARG PCAP_PIPELINE_IGNORE_PREEXISTING=false
 ARG ZEEK_PATH=/zeek
 
-ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
+ENV OPENSEARCH_URL $OPENSEARCH_URL
 ENV PCAP_PATH $PCAP_PATH
 ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG
 ENV PCAP_PIPELINE_DEBUG_EXTRA $PCAP_PIPELINE_DEBUG_EXTRA

README.md

@@ -141,7 +141,7 @@ You can then observe that the images have been retrieved by running `docker imag
 $ docker images
 REPOSITORY                                          TAG                 IMAGE ID            CREATED             SIZE
 malcolmnetsec/arkime                                3.2.2               xxxxxxxxxxxx        39 hours ago        683MB
-malcolmnetsec/elasticsearch-od                      3.2.2               xxxxxxxxxxxx        40 hours ago        690MB
+malcolmnetsec/opensearch                      3.2.2               xxxxxxxxxxxx        40 hours ago        690MB
 malcolmnetsec/file-monitor                          3.2.2               xxxxxxxxxxxx        39 hours ago        470MB
 malcolmnetsec/file-upload                           3.2.2               xxxxxxxxxxxx        39 hours ago        199MB
 malcolmnetsec/filebeat-oss                          3.2.2               xxxxxxxxxxxx        39 hours ago        555MB
@@ -346,7 +346,7 @@ $ ./scripts/build.sh
 Then, go take a walk or something since it will be a while. When you're done, you can run `docker images` and see you have fresh images for:
 
 * `malcolmnetsec/arkime` (based on `debian:buster-slim`)
-* `malcolmnetsec/elasticsearch-od` (based on `amazon/opendistro-for-elasticsearch`)
+* `malcolmnetsec/opensearch` (based on `amazon/opendistro-for-elasticsearch`)
 * `malcolmnetsec/filebeat-oss` (based on `docker.elastic.co/beats/filebeat-oss`)
 * `malcolmnetsec/file-monitor` (based on `debian:buster-slim`)
 * `malcolmnetsec/file-upload` (based on `debian:buster-slim`)
@@ -458,7 +458,7 @@ Although `install.py` will attempt to automate many of the following configurati
 
 #### <a name="DockerComposeYml"></a>`docker-compose.yml` parameters
 
-Edit `docker-compose.yml` and search for the `ES_JAVA_OPTS` key. Edit the `-Xms4g -Xmx4g` values, replacing `4g` with a number that is half of your total system memory, or just under 32 gigabytes, whichever is less. So, for example, if I had 64 gigabytes of memory I would edit those values to be `-Xms31g -Xmx31g`. This indicates how much memory can be allocated to the Elasticsearch heaps. For a pleasant experience, I would suggest not using a value under 10 gigabytes. Similar values can be modified for Logstash with `LS_JAVA_OPTS`, where using 3 or 4 gigabytes is recommended.
+Edit `docker-compose.yml` and search for the `OPENSEARCH_JAVA_OPTS` key. Edit the `-Xms4g -Xmx4g` values, replacing `4g` with a number that is half of your total system memory, or just under 32 gigabytes, whichever is less. So, for example, if I had 64 gigabytes of memory I would edit those values to be `-Xms31g -Xmx31g`. This indicates how much memory can be allocated to the Elasticsearch heaps. For a pleasant experience, I would suggest not using a value under 10 gigabytes. Similar values can be modified for Logstash with `LS_JAVA_OPTS`, where using 3 or 4 gigabytes is recommended.
 
 Various other environment variables inside of `docker-compose.yml` can be tweaked to control aspects of how Malcolm behaves, particularly with regards to processing PCAP files and Zeek logs. The environment variables of particular interest are located near the top of that file under **Commonly tweaked configuration options**, which include:
 
@@ -1825,7 +1825,7 @@ Pulling zeek          ... done
 user@host:~/Malcolm$ docker images
 REPOSITORY                                          TAG                 IMAGE ID            CREATED             SIZE
 malcolmnetsec/arkime                                3.2.2               xxxxxxxxxxxx        39 hours ago        683MB
-malcolmnetsec/elasticsearch-od                      3.2.2               xxxxxxxxxxxx        40 hours ago        690MB
+malcolmnetsec/opensearch                      3.2.2               xxxxxxxxxxxx        40 hours ago        690MB
 malcolmnetsec/file-monitor                          3.2.2               xxxxxxxxxxxx        39 hours ago        470MB
 malcolmnetsec/file-upload                           3.2.2               xxxxxxxxxxxx        39 hours ago        199MB
 malcolmnetsec/filebeat-oss                          3.2.2               xxxxxxxxxxxx        39 hours ago        555MB