diff --git a/README.md b/README.md index 9c61150..8f90c14 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# edcop-tools +# edcop-tools - DEPRECATED: tools now have their own repos under sealingtech/EDCOP-$TOOL EDCOP-Tools are meant to be deployed on the EDCOP platform. EDCOP-Tools is a compilation of DCO tools which are built on Docker and Kubernetes. The goal of this project is to build an easily deployable and manageable DCO infrastructure which is fully integrated using DevOps practices. This is still an early prototype at this phase. View the website diff --git a/bro/README.md b/bro/README.md index 9bf7646..d6a9e16 100644 --- a/bro/README.md +++ b/bro/README.md @@ -6,7 +6,6 @@ Table of Contents * [Configuration Guide](#configuration-guide) * [Image Repository](#image-repository) * [Networks](#networks) - * [Persistent Storage](#persistent-storage) * [Node Selector](#node-selector) * [Bro Configuration](#bro-configuration) * [Resource Limits](#resource-limits) @@ -21,7 +20,7 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing these values, make sure you use the full repository name. +By default, Bro is pulled from EDCOP's official repo and the rest of the applications use their respective official images hosted on Docker's hub. If you're changing these values, make sure you use the full repository name. ``` images: @@ -51,21 +50,6 @@ passive 1d inline-1 1d inline-2 1d ``` - -## Persistent Storage - -These values tell Kubernetes where Bro's logs should be stored on the -host for persistent storage. The *spool* option is for Bro's current -logs and the *logs* option is for Bro's old logs. By default, these values are set to */var/EDCOP/data/logs/bro* but should be changed according to your logical volume setup. - -``` -volumes: - logs: - spool: - hostPath: /var/EDCOP/data/logs/bro/spool - logs: - hostPath: /var/EDCOP/data/logs/bro/logs -``` ## Node Selector @@ -80,9 +64,9 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` ## Bro Configuration @@ -95,6 +79,9 @@ You can set limits on Bro to ensure it doesn't use more CPU/memory space than ne ``` broConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 4G @@ -124,6 +111,9 @@ logstashConfig: maxJvmHeap: 4g pipelineOutputWorkers: 2 pipelineBatchSize: 150 + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G @@ -135,6 +125,9 @@ Redis is also included in the Daemonset for the same reasons Logstash is. Curren ``` redisConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G diff --git a/bro/helm/Chart.yaml b/bro/helm/Chart.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/templates/_helpers.tpl b/bro/helm/templates/_helpers.tpl old mode 100755 new mode 100644 diff --git a/bro/helm/templates/bro-daemonset.yaml b/bro/helm/templates/bro-daemonset.yaml old mode 100755 new mode 100644 index f94c41a..d78f1d3 --- a/bro/helm/templates/bro-daemonset.yaml +++ b/bro/helm/templates/bro-daemonset.yaml @@ -84,8 +84,6 @@ spec: - name: filebeat image: {{ .Values.images.filebeat }} imagePullPolicy: Always - securityContext: - privileged: true volumeMounts: - mountPath: /bro/ name: bro-spool diff --git a/bro/helm/templates/bro-etc-config.yaml b/bro/helm/templates/bro-etc-config.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/templates/bro-site-config.yaml b/bro/helm/templates/bro-site-config.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/templates/filebeat-bro-config.yaml b/bro/helm/templates/filebeat-bro-config.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/templates/logstash-bro-config.yaml b/bro/helm/templates/logstash-bro-config.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/templates/logstash-bro-pipeline.yaml b/bro/helm/templates/logstash-bro-pipeline.yaml old mode 100755 new mode 100644 diff --git a/bro/helm/values.yaml b/bro/helm/values.yaml old mode 100755 new mode 100644 diff --git a/elasticsearch/README.md b/elasticsearch/README.md index 6f19624..d8f8f2b 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -9,6 +9,7 @@ Table of Contents * [Persistent Volume Storage](#persistent-volume-storage) * [Node Selector](#node-selector) * [Elasticsearch Configuration](#elasticsearch-configuration) + * [General](#general) * [Environment](#environment) * [Resource Limits](#resource-limits) * [Curator Configuration](#curator-configuration) @@ -23,12 +24,12 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing these values, make sure you include the full repository name. +By default, Elasticsearch is pulled from Elastic's official repository and the Curator is pulled from a customized image hosted on Docker's hub. If you're changing these values, make sure you include the full repository name. ``` images: - elasticsearch: edcop-master:5000/elasticsearch - curator: edcop-master:5000/curator + elasticsearch: docker.elastic.co/elasticsearch/elasticsearch:6.2.4 + curator: bobrik/curator ``` ## Networks @@ -53,11 +54,11 @@ inline-2 1d ## Persistent Volume Storage -These values tell Kubernetes where Elasticsearch's index data should be stored on the host for persistent storage. By default, this value is set to */var/EDCOP/data/esdata* but should be changed according to your logical volume setup. +These values tell Kubernetes where Elasticsearch's index data should be stored on the host for persistent storage. By default, this value is set to */EDCOP/bulk/esdata* but should be changed according to your logical volume setup. ``` volumes: - data: /var/EDCOP/data/esdata + data: /EDCOP/bulk/esdata ``` ## Node Selector @@ -74,14 +75,30 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` ## Elasticsearch Configuration -Elasticsearch is deployed as a daemonset spread across all of the worker nodes in a single cluster. These instances point to the master deployment that should be on your Kubernetes master node. +Elasticsearch is deployed as a statefulset spread across all of the worker nodes in a single cluster. These instances point to the master deployment that should be on your Kubernetes master node. + +### General + +In order to prevent permission issues, elasticsearch is required to run as a different user and that user should own the volume directory you specified above. This user must e created beforehand and should only have access to this directory/subdirectories for security purposes. Enter the UID of this user in the space below: + +``` +elasticsearchConfig: + runAsUser: 2000 +``` + +Since Elasticsearch's workers are run as statefulsets, you need to specify how many instances you want to maintain. By default, this value is 3, but should be scaled to include the number of worker nodes you have. Do not include the master as one instance because it is deployed in a seperate deployment that only runs on the master. + +``` +elasticsearchConfig: + workerNodes: 3 +``` ### Environment @@ -96,11 +113,14 @@ elasticsearchConfig: ### Resource Limits -The second part of Elasticsearch's configuration allows you to limit the CPU and memory usage. Elasticsearch recommends memory to be capped at a 32GB maximum per their instructions available [here](https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html). +The second part of Elasticsearch's configuration allows you to limit the CPU and memory usage. The request values must be smaller than the limit values and are set low by default to accomodate VMs. Elasticsearch recommends memory to be capped at a 32GB maximum per their instructions available [here](https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html). ``` elasticsearchConfig: ... + requests: + cpu: 100m + memory: 64Mi limits: cpu: 12 memory: 32Gi diff --git a/elasticsearch/helm/Chart.yaml b/elasticsearch/helm/Chart.yaml index cb930d5..5516fc4 100644 --- a/elasticsearch/helm/Chart.yaml +++ b/elasticsearch/helm/Chart.yaml @@ -4,5 +4,5 @@ home: https://github.com/sealingtech/EDCOP version: 0.1.0 description: EDCOP Elasticsearch Chart details: - This Chart provides an Elasticsearch daemonset deployed onto worker nodes within the EDCOP project. + This Chart provides an Elasticsearch statefulset deployed onto worker nodes within the EDCOP project. icon: http://www.elastic.co/assets/bltd74b9a6e0e479359/icon-white-circle-elasticsearch.png diff --git a/elasticsearch/helm/templates/curator-config.yaml b/elasticsearch/helm/templates/curator-config.yaml index c2189f6..643a71d 100644 --- a/elasticsearch/helm/templates/curator-config.yaml +++ b/elasticsearch/helm/templates/curator-config.yaml @@ -1,3 +1,4 @@ +{{ if .Values.curatorConfig.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -56,8 +57,7 @@ data: # Remember, leave a key empty if there is no value. None will be a string, # not a Python "NoneType" client: - hosts: - - data-service + hosts: data-service port: 9200 url_prefix: use_ssl: False @@ -74,3 +74,4 @@ data: logfile: logformat: default blacklist: ['elasticsearch', 'urllib3'] +{{ end }} diff --git a/elasticsearch/helm/templates/curator-cronjob.yaml b/elasticsearch/helm/templates/curator-cronjob.yaml index 5d8e65c..8ce3ac5 100644 --- a/elasticsearch/helm/templates/curator-cronjob.yaml +++ b/elasticsearch/helm/templates/curator-cronjob.yaml @@ -1,3 +1,4 @@ +{{ if .Values.curatorConfig.enabled }} apiVersion: batch/v1beta1 kind: CronJob metadata: @@ -7,7 +8,9 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} spec: - schedule: {{ .Values.curatorConfig.cronjob_schedule }} + schedule: "{{ .Values.curatorConfig.cronjob_schedule }}" + successfulJobsHistoryLimit: {{ .Values.curatorConfig.successfulJobsLimit }} + failedJobsHistoryLimit: {{ .Values.curatorConfig.failedPodsLimit }} jobTemplate: spec: template: @@ -18,8 +21,8 @@ spec: - name: curator image: {{ .Values.images.curator }} imagePullPolicy: Always + command: ["curator"] args: - - curator - --config - /etc/config/config.yml - /etc/config/actions.yml @@ -27,7 +30,10 @@ spec: - name: curator-config mountPath: /etc/config volumes: - - name: curator-config - configMap: - name: {{ template "elasticsearch.fullname" . }}-curator-config + - name: curator-config + configMap: + name: {{ template "elasticsearch.fullname" . }}-curator-config restartPolicy: OnFailure + nodeSelector: + nodetype: {{ .Values.nodeSelector.master }} +{{ end }} diff --git a/elasticsearch/helm/templates/elasticsearch-config.yaml b/elasticsearch/helm/templates/elasticsearch-config.yaml index 00ca2b8..a5c643a 100644 --- a/elasticsearch/helm/templates/elasticsearch-config.yaml +++ b/elasticsearch/helm/templates/elasticsearch-config.yaml @@ -99,5 +99,8 @@ data: xpack.security.enabled: false xpack.watcher.enabled: false xpack.monitoring.enabled: true + # --------------------------------- Snapshots ---------------------------------- + # Location of NFS storage for snapshots + path.repo: ["/var/EDCOP/snapshots"] diff --git a/elasticsearch/helm/templates/es-client.yaml b/elasticsearch/helm/templates/es-client.yaml index 604e1e1..b57ac0f 100644 --- a/elasticsearch/helm/templates/es-client.yaml +++ b/elasticsearch/helm/templates/es-client.yaml @@ -1,5 +1,5 @@ -apiVersion: extensions/v1beta1 -kind: DaemonSet +apiVersion: apps/v1 +kind: StatefulSet metadata: name: {{ template "elasticsearch.fullname" . }} labels: @@ -9,6 +9,11 @@ metadata: component: {{ template "elasticsearch.name" . }} role: client spec: + serviceName: "data-service" + replicas: {{ .Values.elasticsearchConfig.workerNodes }} + selector: + matchLabels: + component: {{ template "elasticsearch.name" . }} template: metadata: name: {{ template "elasticsearch.name" . }} @@ -23,6 +28,8 @@ spec: containers: - name: es-client image: {{ .Values.images.elasticsearch }} + securityContext: + runAsUser: {{ .Values.elasticsearchConfig.runAsUser }} env: - name: NAMESPACE valueFrom: @@ -60,6 +67,8 @@ spec: - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml name: elasticsearch-config + - mountPath: /var/EDCOP/snapshots + name: elasticsearch-snapshots volumes: - name: esdata @@ -71,6 +80,9 @@ spec: - name: elasticsearch-config configMap: name: {{ template "elasticsearch.fullname" . }}-config + - name: elasticsearch-snapshots + persistentVolumeClaim: + claimName: {{ template "elasticsearch.name" . }}-snapshots restartPolicy: Always dnsPolicy: ClusterFirst nodeSelector: diff --git a/elasticsearch/helm/templates/es-master.yaml b/elasticsearch/helm/templates/es-master.yaml index 9f7bf56..51e108f 100644 --- a/elasticsearch/helm/templates/es-master.yaml +++ b/elasticsearch/helm/templates/es-master.yaml @@ -24,6 +24,8 @@ spec: containers: - name: es-master image: {{ .Values.images.elasticsearch }} + securityContext: + runAsUser: {{ .Values.elasticsearchConfig.runAsUser }} env: - name: NAMESPACE valueFrom: @@ -58,6 +60,8 @@ spec: - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml name: elasticsearch-config + - mountPath: /var/EDCOP/snapshots + name: elasticsearch-snapshots volumes: - name: esdata @@ -69,6 +73,9 @@ spec: - name: elasticsearch-config configMap: name: {{ template "elasticsearch.fullname" . }}-config + - name: elasticsearch-snapshots + persistentVolumeClaim: + claimName: {{ template "elasticsearch.name" . }}-snapshots restartPolicy: Always dnsPolicy: ClusterFirst nodeSelector: diff --git a/elasticsearch/helm/templates/snapshot-cronjob.yaml b/elasticsearch/helm/templates/snapshot-cronjob.yaml new file mode 100644 index 0000000..49586ac --- /dev/null +++ b/elasticsearch/helm/templates/snapshot-cronjob.yaml @@ -0,0 +1,35 @@ +{{ if .Values.snapshotConfig.enabled }} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ template "elasticsearch.fullname" . }}-snapshot + labels: + app: {{ template "elasticsearch.name" . }}-snapshot + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + schedule: "{{ .Values.snapshotConfig.cronjob_schedule }}" + successfulJobsHistoryLimit: {{ .Values.snapshotConfig.successfulJobsLimit }} + failedJobsHistoryLimit: {{ .Values.snapshotConfig.failedPodsLimit }} + jobTemplate: + spec: + template: + metadata: + name: {{ template "elasticsearch.name" . }}-snapshot + spec: + containers: + - name: snapshot + image: centos + imagePullPolicy: Always + command: ["/snapshot.sh"] + volumeMounts: + - name: snapshot-config + mountPath: /snapshot.sh + subPath: snapshot.sh + volumes: + - name: snapshot-script + configMap: + name: {{ template "elasticsearch.fullname" . }}-snapshot-script + defaultMode: 0744 + restartPolicy: OnFailure +{{ end }} diff --git a/elasticsearch/helm/templates/snapshot-init-job.yaml b/elasticsearch/helm/templates/snapshot-init-job.yaml new file mode 100644 index 0000000..8054709 --- /dev/null +++ b/elasticsearch/helm/templates/snapshot-init-job.yaml @@ -0,0 +1,32 @@ +{{ if .Values.snapshotConfig.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "elasticsearch.fullname" . }}-snapshot-init-job + labels: + app: {{ template "elasticsearch.name" . }}-snapshot + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + template: + metadata: + name: {{ template "elasticsearch.name" . }}-snapshot-job + annotations: + "helm.sh/hook": "post-install" # Run after all other services start + containers: + - name: snapshot-init-job + image: centos + imagePullPolicy: Always + command: ["/snapshot-init.sh"] + volumeMounts: + - name: snapshot-init-script + mountPath: /snapshot-init.sh + subPath: snapshot-init.sh + volumes: + - name: snapshot-init-script + configMap: + name: {{ template "elasticsearch.fullname" . }}-snapshot-init-script + defaultMode: 0744 + restartPolicy: OnFailure +{{ end }} + diff --git a/elasticsearch/helm/templates/snapshot-init-script.yaml b/elasticsearch/helm/templates/snapshot-init-script.yaml new file mode 100644 index 0000000..beb1d00 --- /dev/null +++ b/elasticsearch/helm/templates/snapshot-init-script.yaml @@ -0,0 +1,44 @@ +{{ if .Values.snapshotConfig.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "elasticsearch.fullname" . }}-snapshot-init-script +data: + snapshot-init.sh: | + #!/bin/bash + # Script to create Elasticsearch's snapshot index if it's not there already + + # Need jq for checking + yum install -y epel-release + yum install -y jq + + # Wait for Elasticsearch to start + echo "Waiting for Elasticsearch to start..." + ESSTATUS="" + until [ "$ESSTATUS" == "green" ]; do + ESSTATUS=$(curl -X --head "data-service:9200/_cluster/health?wait_for_status=green" | jq --raw-output '.status') + done + + # Check to see if the index exists + echo "Checking to see if Snapshot index exists..." + INDEXSTATUS=$(curl -X --head "data-service:9200/_snapshot/es_6x_snapshots" | jq --raw-output '.status') + + # If not, create the index + if [ "$INDEXSTATUS" = "404" ] + then + echo "Snapshot index not found, creating..." + curl -X PUT "data-service:9200/_snapshot/es_6x_snapshots" -H 'Content-Type: application/json' -d' + { + "type": "fs", + "settings": { + "location": "/var/EDCOP/snapshots", + "compress": true + } + } + ' + else + echo "Snapshot index already exists, exiting..." + exit + fi +{{ end }} + diff --git a/elasticsearch/helm/templates/snapshot-pvc.yaml b/elasticsearch/helm/templates/snapshot-pvc.yaml new file mode 100644 index 0000000..b3f3ca9 --- /dev/null +++ b/elasticsearch/helm/templates/snapshot-pvc.yaml @@ -0,0 +1,17 @@ +{{ if .Values.snapshotConfig.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + annotations: {} + labels: + created-by: {{ template "elasticsearch.name" . }} + name: {{ template "elasticsearch.name" . }}-snapshots + name: {{ template "elasticsearch.name" . }}-snapshots + namespace: default +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.snapshotConfig.pvc.storage }} +{{ end }} diff --git a/elasticsearch/helm/templates/snapshot-script.yaml b/elasticsearch/helm/templates/snapshot-script.yaml new file mode 100644 index 0000000..39b3ce6 --- /dev/null +++ b/elasticsearch/helm/templates/snapshot-script.yaml @@ -0,0 +1,26 @@ +{{ if .Values.snapshotConfig.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "elasticsearch.fullname" . }}-snapshot-script +data: + snapshot.sh: | + #!/bin/bash + # Script to create Elasticsearch snapshots + + # Need jq for checking + yum install -y epel-release + yum install -y jq + + # Check to make sure Elasticsearch is alive + echo "Checking to see if Elasticsearch is started..." + ESSTATUS="" + until [ "$ESSTATUS" == "green" ]; do + ESSTATUS=$(curl -X --head "data-service:9200/_cluster/health?wait_for_status=green" | jq --raw-output '.status') + done + + DATE=`date +%m-%d-%Y` + DATE="${DATE//-/_}" + + curl -X PUT "data-service:9200/_snapshot/es_6x_snapshots/snapshot_$DATE?wait_for_completion=true" +{{ end }} diff --git a/elasticsearch/helm/values.yaml b/elasticsearch/helm/values.yaml index acb95c0..d8b5b3b 100644 --- a/elasticsearch/helm/values.yaml +++ b/elasticsearch/helm/values.yaml @@ -13,6 +13,10 @@ nodeSelector: client: worker master: master elasticsearchConfig: + # runAsUser refers to the UID of the user who owns the persistent data location specified above + runAsUser: 2000 + # Nodes refers to the number of worker nodes you have + workerNodes: 3 env: clustername: edcop javaopts: -Xms16g -Xmx16g @@ -23,9 +27,25 @@ elasticsearchConfig: limits: cpu: 4 memory: 32Gi +snapshotConfig: + # Set to 'true' to enable it, false to disable. + enabled: true + # Input how often snapshots should be taken in cronjob format + cronjob_schedule: "1 0 * * *" + # Input how many failed/successful pods should be kept after completion (0 for none) + failedPodsLimit: 1 + successfulJobsLimit: 1 + # Amount of NFS storage to give to snapshots of elasticsearch + pvc: + storage: 10Gi curatorConfig: + # Set to 'true' to enable it, false to disable. + enabled: true # Input how often the curator should run in cronjob format - cronjob_schedule: 1 0 * * * + cronjob_schedule: "1 0 * * *" + # Input how many failed/successful pods should be kept after completion (0 for none) + failedPodsLimit: 1 + successfulJobsLimit: 1 actions: close: # (True or False) Set to False to enable this feature. diff --git a/kibana/README.md b/kibana/README.md index 0f38864..a7540d7 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -7,6 +7,7 @@ Table of Contents * [Image Repository](#image-repository) * [Networks](#networks) * [Node Selector](#node-selector) + * [Ingress](#ingress) # Configuration Guide @@ -15,11 +16,11 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing this value, make sure you use the full repository name. +By default, images are pulled from Elastic's official repository. If you're changing this value, make sure you use the full repository name. ``` image: - kibana: edcop-master:5000/kibana + kibana: docker.elastic.co/kibana/kibana:6.2.4 ``` ## Networks @@ -55,7 +56,15 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` + +## Ingress + +In order to serve web traffic to the GUIs provided by the tools, we use Traefik in conjuction with Kubernetes ingress objects. This value should be the FQDN of your EDCOP host. By default, Kibana will be available at $FQDN/kibana/ + +``` +ingress: + host: physial.edcop.io diff --git a/moloch/README.md b/moloch/README.md new file mode 100644 index 0000000..cabb23c --- /dev/null +++ b/moloch/README.md @@ -0,0 +1,140 @@ +# EDCOP Moloch Guide + +Table of Contents +----------------- + +* [Configuration Guide](#configuration-guide) + * [Image Repository](#image-repository) + * [Networks](#networks) + * [Persistent Storage](#persistent-storage) + * [Node Selector](#node-selector) + * [Ingress](#ingress) + * [Moloch Configuration](#moloch-configuration) + * [Performance](#performance) + * [Environment Variables](#environment-variables) + * [Resource Limits](#resource-limits) + +# Configuration Guide + +Within this configuration guide, you will find instructions for modifying Moloch's helm chart. All changes should be made in the *values.yaml* file. +Please share any bugs or features requests via GitHub issues. + +## Image Repository + +By default, the image is pulled from miked235/moloch, which is a customized Moloch image with the ability to specify viewer/capture only nodes. This value should not be changed because the configuration depends on the environment variables specified in the yamls. + +``` +images: + moloch: miked235/moloch +``` + +## Networks + +Moloch only uses 2 interfaces because it can only be deployed in passive mode to record traffic. By default, these interfaces are named *calico* and *passive*. + +``` +networks: + overlay: calico + passive: passive +``` + +To find the names of your networks, use the following command: + +``` +# kubectl get networks +NAME AGE +calico 1d +passive 1d +inline-1 1d +inline-2 1d +``` + +## Persistent Storage + +These values tell Kubernetes where Moloch's PCAPs and logs should be stored on the host for persistent storage. The *raw* option is for Moloch's raw PCAP files and the *logs* option is for Moloch's capture/viewer logs. By default, these values are set to */bulk/EDCOP/moloch/* but should be changed according to your logical volume setup. + +``` +volumes: + logs: /bulk/EDCOP/moloch/logs + raw: /bulk/EDCOP/moloch/raw +``` + +## Node Selector + +This value tells Kubernetes which hosts the daemonset and statefulset should be deployed to by using labels given to the hosts. The viewer nodes run on the master while the capture nodes run on the workers. Hosts without the defined label will not receive pods. + +``` +nodeSelector: + worker: worker + viewer: master +``` + +To find out what labels your hosts have, please use the following: +``` +# kubectl get nodes --show-labels +NAME STATUS ROLES AGE VERSION LABELS +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion +``` + +## Ingress + +In order to serve web traffic to the GUIs provided by the tools, we use Traefik in conjuction with Kubernetes ingress objects. This value should be the FQDN of your EDCOP host. By default, Moloch will be available at $FQDN/moloch/ + +``` +ingress: + host: physial.edcop.io +``` + +## Moloch Configuration + +Moloch is used as a FPCAP solution, so some configuration is required for optimal performance. Clusters that run Moloch will need 2 networks: an overlay and passive tap network. + +### Performance + +Before tweaking Moloch's performance, you need to define how many instances should be run. The value below should be equal to the number of *worker* nodes you have. Unforuntaley, there is no Statefulset-Daemonset, so we're stuck defining the number of nodes we need until there is a better way. + +``` +molochConfig: + workerNodes: 3 +``` + +Moloch allows you to set limits on many different performance settings, but the ones included in the ```values.yaml``` are the most important. Before configuring these values, you should read Moloch's best practices at https://github.com/aol/moloch/wiki/Settings#High_Performance_Settings. By default, these values are set to Moloch's recommended settings. + +``` +molochConfig: + performance: + maxStreams: 1000000 + maxPacketsInQueue: 200000 + maxPackets: 10000 + packetThreads: 5 + pcapWriteSize: 262143 + tpacketv3Threads: 2 +``` + +### Environment Variables + +In order to make Moloch more secure, you need to set a couple of passwords for Moloch's data transfer and access to its viewer. You can set the cluster and encrypt passwords to something random, but the admin password will be used to access the web interface as the admin superuser. You could use something like pwgen to create random passwords, but this isn't necessary. + +``` +molochConfig: + env: + adminpw: supersecretpw + clusterpw: anothersupersecretpw + encryptpw: randencryptpw +``` + +### Resource Limits + +You can set limits on Moloch to ensure it doesn't use more CPU/memory space than necessary. Finding the right balance can be tricky, so some testing may be required. + +``` +molochConfig: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 4G +``` diff --git a/moloch/container/Dockerfile b/moloch/container/Dockerfile new file mode 100644 index 0000000..610a4c9 --- /dev/null +++ b/moloch/container/Dockerfile @@ -0,0 +1,34 @@ +FROM centos:latest + +RUN yum -y update && yum -y install epel-release && \ + yum -y install jq wget nodejs bzip2 curl net-tools fontconfig freetype freetype-devel fontconfig-devel libyaml-devel libpcap-devel libstdc++ ethtool pcre tcpdump pcre-devel libyaml pkgconfig flex bison gcc-c++ zlib-devel e2fsprogs-devel openssl-devel file-devel make gettext libuuid-devel perl-JSON bzip2-libs bzip2-devel perl-libwww-perl libpng-devel xz libffi-devel GeoIP vim && \ + wget https://files.molo.ch/builds/centos-7/moloch-1.1.1-1.x86_64.rpm && \ + rpm -i moloch-1.1.1-1.x86_64.rpm + +ADD scripts /data/moloch/ + +ENV ES_HOST=elasticsearch \ + NETWORK_INTERFACE=eth0 \ + CLUSTER_PW=secretpw \ + ADMIN_PW=supersecretpw \ + # true or false, both cannot be set to false + SENSOR=true \ + VIEWER=true + +RUN chmod +x /data/moloch/*.sh && \ + chmod +x /data/moloch/db/db.pl /data/moloch/*/*.sh && \ + /data/moloch/configmoloch.sh && \ + cd /data/moloch/viewer && \ + ln -s /data/moloch/bin/node /usr/bin/nodejs && \ + /data/moloch/bin/npm update && \ + npm install . && \ + yum clean -y all + +ADD etc/config.ini /data/moloch/etc/config.ini +RUN chmod 755 /data/moloch/etc/config.ini + +EXPOSE 8005 + +WORKDIR /data/moloch + +ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/moloch/container/LICENSE b/moloch/container/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/moloch/container/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/moloch/container/README.md b/moloch/container/README.md new file mode 100644 index 0000000..99a34b4 --- /dev/null +++ b/moloch/container/README.md @@ -0,0 +1,21 @@ +# Moloch Docker Container + +## Docker + +For deploying a Moloch instance with Docker, please have a working Elasticsearch deployment running and a way to access it. You can edit the ```Dockerfile``` environment variables and the ```etc/config.ini``` file to set your configuration settings. + + +The command below uses a Docker link to link the two containers together: + +``` +sudo sysctl -w vm.max_map_count=262144 +sudo docker run -p 9200:9200 -p 9300:9300 -itd --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:6.2.4 +sudo docker run -itd -p 8005:8005 --cap-add NET_RAW --cap-add NET_ADMIN --link elasticsearch:elasticsearch --name moloch moloch +``` + +## Kubernetes + +For deploying multiple instances, it is recommended that you use a statefulset for both Elasticsearch and Moloch to prevent changes in the container/pod states. You may also want to use a config map for the ```config.ini``` file to further customize your deployment. + + +If you're looking for more opensource containerized tools, take a look at https://github.com/sealingtech/EDCOP for a fully automated network security platform that utilizes Docker and Kubernetes for deployments and scaling! diff --git a/moloch/container/etc/config.ini b/moloch/container/etc/config.ini new file mode 100644 index 0000000..67a9b3a --- /dev/null +++ b/moloch/container/etc/config.ini @@ -0,0 +1,362 @@ +# Latest settings documentation: https://github.com/aol/moloch/wiki/Settings +# +# Moloch uses a tiered system for configuration variables. This allows Moloch +# to share one config file for many machines. The ordering of sections in this +# file doesn't matter. +# +# Order of config variables: +# 1st) [optional] The section titled with the node name is used first. +# Moloch will always tag sessions with node: +# 2nd) [optional] If a node has a nodeClass variable, the section titled with +# the nodeClass name is used next. Sessions will be tagged with +# node: which is useful if watching different +# network classes. +# 3rd) The section titled "default" is used last. + +[default] +# Comma seperated list of elasticsearch host:port combinations. If not using a +# elasticsearch VIP, a different elasticsearch node in the cluster can be specified +# for each Moloch node to help spread load on high volume clusters +elasticsearch=elasticsearch:9200 + +# How often to create a new elasticsearch index. hourly,hourly6,daily,weekly,monthly +# Changing the value will cause previous sessions to be unreachable +rotateIndex=daily + +# Cert file to use, comment out to use http instead +# certFile=/data/moloch/etc/moloch.cert + +# File with trusted roots/certs. WARNING! this replaces default roots +# Useful with self signed certs and can be set per node. +# caTrustFile=/data/moloch/etc/roots.cert + +# Private key file to use, comment out to use http instead +# keyFile=/data/moloch/etc/moloch.key + +# Password Hash and S2S secret - Must be in default section. Since elasticsearch +# is wide open by default, we encrypt the stored password hashes with this +# so a malicous person can't insert a working new account. It is also used +# for secure S2S communication. Comment out for no user authentication. +# Changing the value will make all previously stored passwords no longer work. +# Make this RANDOM, you never need to type in +passwordSecret = password + +# Use a different password for S2S communication then passwordSecret. +# Must be in default section. Make this RANDOM, you never need to type in +#serverSecret= + +# HTTP Digest Realm - Must be in default section. Changing the value +# will make all previously stored passwords no longer work +httpRealm = Moloch + +# The base path for Moloch web access. Must end with a / or bad things will happen +# Default: "/" +# webBasePath = /moloch/ + +# Semicolon ';' seperated list of interfaces to listen on for traffic +interface=eth0 + +# The bpf filter of traffic to ignore +#bpf=not port 9200 + +# The yara file name +#yara= + +# Host to connect to for wiseService +#wiseHost=127.0.0.1 + +# Log viewer access requests to a different log file +#accessLogFile = /data/moloch/logs/access.log + +# The directory to save raw pcap files to +pcapDir = /data/moloch/raw + +# The max raw pcap file size in gigabytes, with a max value of 36G. +# The disk should have room for at least 10*maxFileSizeG +maxFileSizeG = 12 + +# The max time in minutes between rotating pcap files. Default is 0, which means +# only rotate based on current file size and the maxFileSizeG variable +#maxFileTimeM = 60 + +# TCP timeout value. Moloch writes a session record after this many seconds +# of inactivity. +tcpTimeout = 600 + +# Moloch writes a session record after this many seconds, no matter if +# active or inactive +tcpSaveTimeout = 720 + +# UDP timeout value. Moloch assumes the UDP session is ended after this +# many seconds of inactivity. +udpTimeout = 30 + +# ICMP timeout value. Moloch assumes the ICMP session is ended after this +# many seconds of inactivity. +icmpTimeout = 10 + +# An aproximiate maximum number of active sessions Moloch/libnids will try +# and monitor +maxStreams = 1000000 + +# Moloch writes a session record after this many packets +maxPackets = 10000 + +# Delete pcap files when free space is lower then this in gigabytes OR it can be +# expressed as a percentage (ex: 5%). This does NOT delete the session records in +# the database. It is recommended this value is between 5% and 10% of the disk. +# Database deletes are done by the db.pl expire script +freeSpaceG = 5% + +# The port to listen on, by default 8005 +viewPort = 8005 + +# The host/ip to listen on, by default 0.0.0.0 which is ALL +#viewHost = localhost + +# By default the viewer process is https://hostname: for each node. +#viewUrl = https://HOSTNAME:8005 + +# Path of the maxmind geoip country file. Download free version from: +# https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country +geoLite2Country = /data/moloch/etc/GeoLite2-Country.mmdb + +# Path of the maxmind geoip ASN file. Download free version from: +# https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN +geoLite2ASN = /data/moloch/etc/GeoLite2-ASN.mmdb + +# Path of the rir assignments file +# https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv +rirFile = /data/moloch/etc/ipv4-address-space.csv + +# Path of the OUI file from whareshark +# https://raw.githubusercontent.com/wireshark/wireshark/master/manuf +ouiFile = /data/moloch/etc/oui.txt + +# User to drop privileges to. The pcapDir must be writable by this user or group below +dropUser=nobody + +# Group to drop privileges to. The pcapDir must be writable by this group or user above +dropGroup=daemon + +# Semicolon ';' seperated list of tags which once capture sets for a session causes the +# remaining pcap from being saved for the session. It is likely that the initial packets +# WILL be saved for the session since tags usually aren't set until after several packets +# Each tag can optionally be followed by a : which specifies how many total packets to save +#dontSaveTags= + +# Header to use for determining the username to check in the database for instead of +# using http digest. Use this if apache or something else is doing the auth. +# Set viewHost to localhost or use iptables +# Might need something like this in the httpd.conf +# RewriteRule .* - [E=ENV_RU:%{REMOTE_USER}] +# RequestHeader set MOLOCH_USER %{ENV_RU}e +#userNameHeader=moloch_user + +# Should we parse extra smtp traffic info +parseSMTP=true + +# Should we parse extra smb traffic info +parseSMB=true + +# Should we parse HTTP QS Values +parseQSValue=false + +# Should we calculate sha256 for bodies +supportSha256=false + +# Only index HTTP request bodies less than this number of bytes */ +maxReqBody=64 + +# Only store request bodies that Utf-8? +config.reqBodyOnlyUtf8 = true + +# Semicolon ';' seperated list of SMTP Headers that have ips, need to have the terminating colon ':' +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: + +# Semicolon ';' seperated list of directories to load parsers from +parsersDir=/data/moloch/parsers + +# Semicolon ';' seperated list of directories to load plugins from +pluginsDir=/data/moloch/plugins + +# Semicolon ';' seperated list of plugins to load and the order to load in +# plugins=tagger.so; netflow.so + +# Plugins to load as root, usually just readers +#rootPlugins=reader-pfring; reader-daq.so + +# Semicolon ';' seperated list of viewer plugins to load and the order to load in +# viewerPlugins=wise.js + +# NetFlowPlugin +# Input device id, 0 by default +#netflowSNMPInput=1 +# Outout device id, 0 by default +#netflowSNMPOutput=2 +# Netflow version 1,5,7 supported, 7 by default +#netflowVersion=1 +# Semicolon ';' seperated list of netflow destinations +#netflowDestinations=localhost:9993 + +# Specify the max number of indices we calculate spidata for. +# ES will blow up if we allow the spiData to search too many indices. +spiDataMaxIndices=4 + +# Uncomment the following to allow direct uploads. This is experimental +#uploadCommand=/data/moloch/bin/moloch-capture --copy -n {NODE} -r {TMPFILE} -c {CONFIG} {TAGS} + +# Title Template +# _cluster_ = ES cluster name +# _userId_ = logged in User Id +# _userName_ = logged in User Name +# _page_ = internal page name +# _expression_ = current search expression if set, otherwise blank +# _-expression_ = " - " + current search expression if set, otherwise blank, prior spaces removed +# _view_ = current view if set, otherwise blank +# _-view_ = " - " + current view if set, otherwise blank, prior spaces removed +#titleTemplate=_cluster_ - _page_ _-view_ _-expression_ + +# Number of threads processing packets +packetThreads=2 + +# ADVANCED - Semicolon ';' seperated list of files to load for config. Files are loaded +# in order and can replace values set in this file or previous files. +#includes= + +# ADVANCED - How is pcap written to disk +# simple = use O_DIRECT if available, writes in pcapWriteSize chunks, +# a file per packet thread. +# simple-nodirect = don't use O_DIRECT. Required for zfs and others +pcapWriteMethod=simple + +# ADVANCED - Buffer size when writing pcap files. Should be a multiple of the raid 5 or xfs +# stripe size. Defaults to 256k +pcapWriteSize = 262143 + +# ADVANCED - Number of bytes to bulk index at a time +dbBulkSize = 300000 + +# ADVANCED - Compress requests to ES, reduces ES bandwidth by ~80% at the cost +# of increased CPU. MUST have "http.compression: true" in elasticsearch.yml file +compressES = false + +# ADVANCED - Max number of connections to elastic search +maxESConns = 30 + +# ADVANCED - Max number of es requests outstanding in q +maxESRequests = 500 + +# ADVANCED - Number of packets to ask libnids/libpcap to read per poll/spin +# Increasing may hurt stats and ES performance +# Decreasing may cause more dropped packets +packetsPerPoll = 50000 + +# ADVANCED - Moloch will try to compensate for SYN packet drops by swapping +# the source and destination addresses when a SYN-acK packet was captured first. +# Probably useful to set it false, when running Moloch in wild due to SYN floods. +antiSynDrop = true + +# DEBUG - Write to stdout info every X packets. +# Set to -1 to never log status +logEveryXPackets = 100000 + +# DEBUG - Write to stdout unknown protocols +logUnknownProtocols = false + +# DEBUG - Write to stdout elastic search requests +logESRequests = true + +# DEBUG - Write to stdout file creation information +logFileCreation = true + + +### High Performance settings +# https://github.com/aol/moloch/wiki/Settings#High_Performance_Settings +# magicMode=basic +# pcapReadMethod=tpacketv3 +# tpacketv3NumThreads=2 +# pcapWriteMethod=simple +# pcapWriteSize = 2560000 +# packetThreads=5 +# maxPacketsInQueue = 200000 + +### Low Bandwidth settings +# packetThreads=1 +# pcapWriteSize = 65536 + + +############################################################################## +# Classes of nodes +# Can override most default values, and create a tag call node: +[class1] +freeSpaceG = 10% + +############################################################################## +# Nodes +# Usually just use the hostname before the first dot as the node name +# Can override most default values + +[node1] +nodeClass = class1 +# Might use a different elasticsearch node +elasticsearch=elasticsearchhost1 + +# Uncomment if this node should process the cron queries, only ONE node should process cron queries +# cronQueries = true + +[node2] +nodeClass = class2 +# Might use a different elasticsearch node +elasticsearch=elasticsearchhost2 +# Uses a different interface +interface = eth4 + +############################################################################## +# override-ips is a special section that overrides the MaxMind databases for +# the fields set, but fields not set will still use MaxMind (example if you set +# tags but not country it will use MaxMind for the country) +# Spaces and capitalization is very important. +# IP Can be a single IP or a CIDR +# Up to 10 tags can be added +# +# ip=tag:TAGNAME1;tag:TAGNAME2;country:3LetterUpperCaseCountry;asn:ASN STRING +#[override-ips] +#10.1.0.0/16=tag:ny-office;country:USA;asn:AS0000 This is an ASN + +############################################################################## +# It is now possible to define in the config file extra http/email headers +# to index. They are accessed using the expression http. and +# email. with optional .cnt expressions +# +# Possible config atributes for all headers +# type: (string|integer|ip) = data type (default string) +# count: = index count of items (default false) +# unique: = only record unique items (default true) + +# headers-http-request is used to configure request headers to index +#[headers-http-request] +#referer=type:string;count:true;unique:true + +# headers-http-response is used to configure http response headers to index +#[headers-http-response] +#location=type:string;count:true + +# headers-email is used to configure email headers to index +#[headers-email] +#x-priority=type:integer + + +############################################################################## +# If you have multiple clusters and you want the ability to send sessions +# from one cluster to another either manually or with the cron feature fill out +# this section + +#[moloch-clusters] +#forensics=url:https://viewer1.host.domain:8005;passwordSecret:password4moloch;name:Forensics Cluster +#shortname2=url:http://viewer2.host.domain:8123;passwordSecret:password4moloch;name:Testing Cluster + + + +# WARNING: This is an ini file with sections, most likely you don't want to put a setting here. +# New settings usually go near the top in the [default] section, or in [nodename] sections. diff --git a/moloch/container/scripts/configmoloch.sh b/moloch/container/scripts/configmoloch.sh new file mode 100644 index 0000000..46af0e6 --- /dev/null +++ b/moloch/container/scripts/configmoloch.sh @@ -0,0 +1,10 @@ +# Configuring Moloch Script, uses default values. +# Instead of editing these values, make changes to /etc/config.ini + +/data/moloch/bin/Configure << EOF +$NETWORK_INTERFACE +no +$ES_HOST:9200 +$CLUSTER_PW +EOF + diff --git a/moloch/container/scripts/docker-entrypoint.sh b/moloch/container/scripts/docker-entrypoint.sh new file mode 100644 index 0000000..d1b975c --- /dev/null +++ b/moloch/container/scripts/docker-entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Script to initialize Moloch, add a user, and run the services + +# Check to see if Elasticsearch is reachable +echo "Trying to reach Elasticsearch..." +until $(curl --output /dev/null --fail --silent -X GET "$ES_HOST:9200/_cat/health?v"); do + echo "Couldn't get Elasticsearch at $ES_HOST:9200, are you sure it's reachable?" + sleep 5 +done + +# Check to see if Moloch has been installed before to prevent data loss +STATUS5=$(curl -X --head "$ES_HOST:9200/sequence_v1" | jq --raw-output '.status') +STATUS6=$(curl -X --head "$ES_HOST:9200/sequence_v2" | jq --raw-output '.status') + +# Initialize Moloch if this is the first install +if [ "$STATUS5" = "404" ] && [ "$STATUS6" = "404" ] +then + echo "Initializing Moloch indices..." + echo INIT | /data/moloch/db/db.pl http://$ES_HOST:9200 init + /data/moloch/bin/moloch_add_user.sh admin "Admin User" $ADMIN_PW --admin + /data/moloch/bin/moloch_update_geo.sh +fi + +chmod a+rwx /data/moloch/raw /data/moloch/logs + +# Deploy Moloch as both a sensor and viewer node +if [ "$SENSOR" = "true" ] && [ "$VIEWER" = "true" ] +then + echo "Starting Moloch capture and viewer..." + /data/moloch/bin/moloch_config_interfaces.sh + cd /data/moloch + nohup /data/moloch/bin/moloch-capture -c /data/moloch/etc/config.ini >> /data/moloch/logs/capture.log 2>&1 & + cd /data/moloch/viewer + /data/moloch/bin/node viewer.js -c /data/moloch/etc/config.ini >> /data/moloch/logs/viewer.log 2>&1 +# Sensor only node +elif [ "$SENSOR" = "true" ] +then + echo "Starting Moloch capture..." + /data/moloch/bin/moloch_config_interfaces.sh + cd /data/moloch + /data/moloch/bin/moloch-capture -c /data/moloch/etc/config.ini >> /data/moloch/logs/capture.log 2>&1 +# Viewer only node +elif [ "$VIEWER" = "true" ] +then + echo "Starting Moloch viewer..." + cd /data/moloch/viewer + /data/moloch/bin/node viewer.js -c /data/moloch/etc/config.ini >> /data/moloch/logs/viewer.log 2>&1 +# Error +else + echo "Both SENSOR and VIEWER cannot be set to false, exiting..." + exit +fi + diff --git a/moloch/helm/.DS_Store b/moloch/helm/.DS_Store new file mode 100644 index 0000000..91d87f0 Binary files /dev/null and b/moloch/helm/.DS_Store differ diff --git a/moloch/helm/.helmignore b/moloch/helm/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/moloch/helm/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/moloch/helm/Chart.yaml b/moloch/helm/Chart.yaml new file mode 100644 index 0000000..41efda4 --- /dev/null +++ b/moloch/helm/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +name: moloch +home: https://github.com/sealingtech/EDCOP +version: 0.1.0 +description: EDCOP Moloch Chart +details: + This Chart provides Moloch capture and viewer nodes deployed as statefulsets and deployments, respectively. The viewer nodes are deployed onto master nodes and the capture nodes are deployed to the worker nodes within the EDCOP project. +icon: https://molo.ch/moloch_2x2.png diff --git a/moloch/helm/templates/_helpers.tpl b/moloch/helm/templates/_helpers.tpl new file mode 100644 index 0000000..0937436 --- /dev/null +++ b/moloch/helm/templates/_helpers.tpl @@ -0,0 +1,16 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "moloch.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "moloch.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/moloch/helm/templates/moloch-capture.yaml b/moloch/helm/templates/moloch-capture.yaml new file mode 100644 index 0000000..e6daad0 --- /dev/null +++ b/moloch/helm/templates/moloch-capture.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "moloch.fullname" . }}-capture + labels: + app: {{ template "moloch.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + role: capture +spec: + serviceName: {{ template "moloch.name" . }}-headless + replicas: {{ .Values.molochConfig.workerNodes }} + selector: + matchLabels: + app: {{ template "moloch.name" . }} + role: capture + template: + metadata: + name: {{ template "moloch.name" . }}-capture + labels: + app: {{ template "moloch.name" . }} + role: capture + annotations: + networks: '[ + { "name": {{ .Values.networks.overlay | quote }} }, + { "name": {{ .Values.networks.passive | quote }} } + ]' + spec: + containers: + - name: moloch-capture + image: {{ .Values.images.moloch }} + imagePullPolicy: Always + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - IPC_LOCK + env: + - name: ES_HOST + value: data-service + - name: NETWORK_INTERFACE + value: {{ .Values.networks.passive }} + - name: CLUSTER_PW + value: {{ .Values.molochConfig.env.clusterpw }} + - name: ADMIN_PW + value: {{ .Values.molochConfig.env.adminpw }} + - name: SENSOR + value: "true" + - name: VIEWER + value: "false" + ports: + - name: cport-8005 + containerPort: 8005 + resources: + requests: + cpu: {{ .Values.molochConfig.requests.cpu | quote }} + memory: {{ .Values.molochConfig.requests.memory | quote }} + limits: + cpu: {{ .Values.molochConfig.limits.cpu | quote }} + memory: {{ .Values.molochConfig.limits.memory | quote }} + volumeMounts: + - mountPath: /data/moloch/logs + name: molochlogs + - mountPath: /data/moloch/raw + name: molochraw + - mounthPath: /data/moloch/etc/config.ini + subPath: config.ini + name: moloch-config + volumes: + - name: molochlogs + hostPath: + path: {{ .Values.volumes.logs }} + - name: molochraw + hostPath: + path: {{ .Values.volumes.raw }} + - name: moloch-config + configMap: + name: {{ template "moloch.fullname" . }}-config + restartPolicy: Always + dnsPolicy: ClusterFirst + nodeSelector: + nodetype: {{ .Values.nodeSelector.worker }} diff --git a/moloch/helm/templates/moloch-config.yaml b/moloch/helm/templates/moloch-config.yaml new file mode 100644 index 0000000..84d255e --- /dev/null +++ b/moloch/helm/templates/moloch-config.yaml @@ -0,0 +1,361 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "moloch.fullname" . }}-config +data: + config.ini: | + # Latest settings documentation: https://github.com/aol/moloch/wiki/Settings + # + # Moloch uses a tiered system for configuration variables. This allows Moloch + # to share one config file for many machines. The ordering of sections in this + # file doesn't matter. + # + # Order of config variables: + # 1st) [optional] The section titled with the node name is used first. + # Moloch will always tag sessions with node: + # 2nd) [optional] If a node has a nodeClass variable, the section titled with + # the nodeClass name is used next. Sessions will be tagged with + # node: which is useful if watching different + # network classes. + # 3rd) The section titled "default" is used last. + [default] + # Comma seperated list of elasticsearch host:port combinations. If not using a + # elasticsearch VIP, a different elasticsearch node in the cluster can be specified + # for each Moloch node to help spread load on high volume clusters + elasticsearch=data-service:9200 + + # How often to create a new elasticsearch index. hourly,hourly6,daily,weekly,monthly + # Changing the value will cause previous sessions to be unreachable + rotateIndex=daily + # Cert file to use, comment out to use http instead + # certFile=/data/moloch/etc/moloch.cert + + # File with trusted roots/certs. WARNING! this replaces default roots + # Useful with self signed certs and can be set per node. + # caTrustFile=/data/moloch/etc/roots.cert + + # Private key file to use, comment out to use http instead + # keyFile=/data/moloch/etc/moloch.key + + # Password Hash and S2S secret - Must be in default section. Since elasticsearch + # is wide open by default, we encrypt the stored password hashes with this + # so a malicous person can't insert a working new account. It is also used + # for secure S2S communication. Comment out for no user authentication. + # Changing the value will make all previously stored passwords no longer work. + # Make this RANDOM, you never need to type in + passwordSecret = {{ .Values.molochConfig.env.encryptpw }} + + # Use a different password for S2S communication then passwordSecret. + # Must be in default section. Make this RANDOM, you never need to type in + #serverSecret= + + # HTTP Digest Realm - Must be in default section. Changing the value + # will make all previously stored passwords no longer work + httpRealm = Moloch + + # The base path for Moloch web access. Must end with a / or bad things will happen + # Default: "/" + webBasePath = /moloch/ + + # Semicolon ';' seperated list of interfaces to listen on for traffic + interface=net0 + + # The bpf filter of traffic to ignore + #bpf=not port 9200 + + # The yara file name + #yara= + + # Host to connect to for wiseService + #wiseHost=127.0.0.1 + + # Log viewer access requests to a different log file + #accessLogFile = /data/moloch/logs/access.log + + # The directory to save raw pcap files to + pcapDir = /data/moloch/raw + + # The max raw pcap file size in gigabytes, with a max value of 36G. + # The disk should have room for at least 10*maxFileSizeG + maxFileSizeG = 12 + + # The max time in minutes between rotating pcap files. Default is 0, which means + # only rotate based on current file size and the maxFileSizeG variable + #maxFileTimeM = 60 + # TCP timeout value. Moloch writes a session record after this many seconds + # of inactivity. + tcpTimeout = 600 + + # Moloch writes a session record after this many seconds, no matter if + # active or inactive + tcpSaveTimeout = 720 + + # UDP timeout value. Moloch assumes the UDP session is ended after this + # many seconds of inactivity. + udpTimeout = 30 + + # ICMP timeout value. Moloch assumes the ICMP session is ended after this + # many seconds of inactivity. + icmpTimeout = 10 + + # An aproximiate maximum number of active sessions Moloch/libnids will try + # and monitor + maxStreams = {{ .Values.molochConfig.performance.maxStreams }} + + # Moloch writes a session record after this many packets + maxPackets = {{ .Values.molochConfig.performance.maxPackets }} + + # Delete pcap files when free space is lower then this in gigabytes OR it can be + # expressed as a percentage (ex: 5%). This does NOT delete the session records in + # the database. It is recommended this value is between 5% and 10% of the disk. + # Database deletes are done by the db.pl expire script + freeSpaceG = 5% + + # The port to listen on, by default 8005 + viewPort = 8005 + + # The host/ip to listen on, by default 0.0.0.0 which is ALL + #viewHost = localhost + + # By default the viewer process is https://hostname: for each node. + #viewUrl = https://HOSTNAME:8005 + + # Path of the maxmind geoip country file. Download free version from: + # https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country + geoLite2Country = /data/moloch/etc/GeoLite2-Country.mmdb + + # Path of the maxmind geoip ASN file. Download free version from: + # https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN + geoLite2ASN = /data/moloch/etc/GeoLite2-ASN.mmdb + + # Path of the rir assignments file + # https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv + rirFile = /data/moloch/etc/ipv4-address-space.csv + + # Path of the OUI file from whareshark + # https://raw.githubusercontent.com/wireshark/wireshark/master/manuf + ouiFile = /data/moloch/etc/oui.txt + + # User to drop privileges to. The pcapDir must be writable by this user or group below + dropUser=nobody + + # Group to drop privileges to. The pcapDir must be writable by this group or user above + dropGroup=daemon + + # Semicolon ';' seperated list of tags which once capture sets for a session causes the + # remaining pcap from being saved for the session. It is likely that the initial packets + # WILL be saved for the session since tags usually aren't set until after several packets + # Each tag can optionally be followed by a : which specifies how many total packets to save + #dontSaveTags= + + # Header to use for determining the username to check in the database for instead of + # using http digest. Use this if apache or something else is doing the auth. + # Set viewHost to localhost or use iptables + # Might need something like this in the httpd.conf + # RewriteRule .* - [E=ENV_RU:%{REMOTE_USER}] + # RequestHeader set MOLOCH_USER %{ENV_RU}e + #userNameHeader=moloch_user + + # Should we parse extra smtp traffic info + parseSMTP=true + + # Should we parse extra smb traffic info + parseSMB=true + + # Should we parse HTTP QS Values + parseQSValue=false + + # Should we calculate sha256 for bodies + supportSha256=false + + # Only index HTTP request bodies less than this number of bytes */ + maxReqBody=64 + + # Only store request bodies that Utf-8? + config.reqBodyOnlyUtf8 = true + + # Semicolon ';' seperated list of SMTP Headers that have ips, need to have the terminating colon ':' + smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: + + # Semicolon ';' seperated list of directories to load parsers from + parsersDir=/data/moloch/parsers + + # Semicolon ';' seperated list of directories to load plugins from + pluginsDir=/data/moloch/plugins + + # Semicolon ';' seperated list of plugins to load and the order to load in + # plugins=tagger.so; netflow.so + + # Plugins to load as root, usually just readers + #rootPlugins=reader-pfring; reader-daq.so + + # Semicolon ';' seperated list of viewer plugins to load and the order to load in + # viewerPlugins=wise.js + + # NetFlowPlugin + # Input device id, 0 by default + #netflowSNMPInput=1 + # Outout device id, 0 by default + #netflowSNMPOutput=2 + # Netflow version 1,5,7 supported, 7 by default + #netflowVersion=1 + # Semicolon ';' seperated list of netflow destinations + #netflowDestinations=localhost:9993 + + # Specify the max number of indices we calculate spidata for. + # ES will blow up if we allow the spiData to search too many indices. + spiDataMaxIndices=4 + + # Uncomment the following to allow direct uploads. This is experimental + #uploadCommand=/data/moloch/bin/moloch-capture --copy -n {NODE} -r {TMPFILE} -c {CONFIG} {TAGS} + + # Title Template + # _cluster_ = ES cluster name + # _userId_ = logged in User Id + # _userName_ = logged in User Name + # _page_ = internal page name + # _expression_ = current search expression if set, otherwise blank + # _-expression_ = " - " + current search expression if set, otherwise blank, prior spaces removed + # _view_ = current view if set, otherwise blank + # _-view_ = " - " + current view if set, otherwise blank, prior spaces removed + #titleTemplate=_cluster_ - _page_ _-view_ _-expression_ + + # Number of threads processing packets + packetThreads={{ .Values.molochConfig.performance.packetThreads }} + + # ADVANCED - Semicolon ';' seperated list of files to load for config. Files are loaded + # in order and can replace values set in this file or previous files. + #includes= + + # ADVANCED - How is pcap written to disk + # simple = use O_DIRECT if available, writes in pcapWriteSize chunks, a file per packet thread. + # simple-nodirect = don't use O_DIRECT. Required for zfs and others + pcapWriteMethod=simple + + # ADVANCED - Buffer size when writing pcap files. Should be a multiple of the raid 5 or xfs + # stripe size. Defaults to 256k + pcapWriteSize = {{ .Values.molochConfig.performance.pcapWriteSize }} + + # ADVANCED - Number of bytes to bulk index at a time + dbBulkSize = 300000 + + # ADVANCED - Compress requests to ES, reduces ES bandwidth by ~80% at the cost + # of increased CPU. MUST have "http.compression: true" in elasticsearch.yml file + compressES = false + + # ADVANCED - Max number of connections to elastic search + maxESConns = 30 + + # ADVANCED - Max number of es requests outstanding in q + maxESRequests = 500 + + # ADVANCED - Number of packets to ask libnids/libpcap to read per poll/spin + # Increasing may hurt stats and ES performance + # Decreasing may cause more dropped packets + packetsPerPoll = 50000 + + # ADVANCED - Moloch will try to compensate for SYN packet drops by swapping + # the source and destination addresses when a SYN-acK packet was captured first. + # Probably useful to set it false, when running Moloch in wild due to SYN floods. + antiSynDrop = false + + # DEBUG - Write to stdout info every X packets. + # Set to -1 to never log status + logEveryXPackets = 100000 + + # DEBUG - Write to stdout unknown protocols + logUnknownProtocols = false + + # DEBUG - Write to stdout elastic search requests + logESRequests = true + + # DEBUG - Write to stdout file creation information + logFileCreation = true + + + ### High Performance settings + # https://github.com/aol/moloch/wiki/Settings#High_Performance_Settings + magicMode=basic + pcapReadMethod=tpacketv3 + tpacketv3NumThreads={{ .Values.molochConfig.performance.tpacketv3Threads }} + maxPacketsInQueue ={{ .Values.molochConfig.performance.maxPacketsInQueue }} + + ### Low Bandwidth settings + # packetThreads=1 + # pcapWriteSize = 65536 + + + ############################################################################## + # Classes of nodes + # Can override most default values, and create a tag call node: + [class1] + freeSpaceG = 10% + + ############################################################################## + # Nodes + # Usually just use the hostname before the first dot as the node name + # Can override most default values + + [node1] + nodeClass = class1 + # Might use a different elasticsearch node + elasticsearch=elasticsearchhost1 + + # Uncomment if this node should process the cron queries, only ONE node should process cron queries + # cronQueries = true + + [node2] + nodeClass = class2 + # Might use a different elasticsearch node + elasticsearch=elasticsearchhost2 + # Uses a different interface + interface = eth4 + + ############################################################################## + # override-ips is a special section that overrides the MaxMind databases for + # the fields set, but fields not set will still use MaxMind (example if you set + # tags but not country it will use MaxMind for the country) + # Spaces and capitalization is very important. + # IP Can be a single IP or a CIDR + # Up to 10 tags can be added + # + # ip=tag:TAGNAME1;tag:TAGNAME2;country:3LetterUpperCaseCountry;asn:ASN STRING + #[override-ips] + #10.1.0.0/16=tag:ny-office;country:USA;asn:AS0000 This is an ASN + + ############################################################################## + # It is now possible to define in the config file extra http/email headers + # to index. They are accessed using the expression http. and + # email. with optional .cnt expressions + # + # Possible config atributes for all headers + # type: (string|integer|ip) = data type (default string) + # count: = index count of items (default false) + # unique: = only record unique items (default true) + + # headers-http-request is used to configure request headers to index + #[headers-http-request] + #referer=type:string;count:true;unique:true + + # headers-http-response is used to configure http response headers to index + #[headers-http-response] + #location=type:string;count:true + + # headers-email is used to configure email headers to index + #[headers-email] + #x-priority=type:integer + + + ############################################################################## + # If you have multiple clusters and you want the ability to send sessions + # from one cluster to another either manually or with the cron feature fill out + # this section + + #[moloch-clusters] + #forensics=url:https://viewer1.host.domain:8005;passwordSecret:password4moloch;name:Forensics Cluster + #shortname2=url:http://viewer2.host.domain:8123;passwordSecret:password4moloch;name:Testing Cluster + + + + # WARNING: This is an ini file with sections, most likely you don't want to put a setting here. + # New settings usually go near the top in the [default] section, or in [nodename] sections. diff --git a/moloch/helm/templates/moloch-headless-svc.yaml b/moloch/helm/templates/moloch-headless-svc.yaml new file mode 100644 index 0000000..fee71ab --- /dev/null +++ b/moloch/helm/templates/moloch-headless-svc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "moloch.name" . }}-headless + namespace: default + labels: + app: {{ template "moloch.name" . }} + role: client +spec: + clusterIP: None + selector: + app: {{ template "moloch.name" . }} + role: client + diff --git a/moloch/helm/templates/moloch-ingress.yaml b/moloch/helm/templates/moloch-ingress.yaml new file mode 100644 index 0000000..794f6e6 --- /dev/null +++ b/moloch/helm/templates/moloch-ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "moloch.name" . }}-ingress + annotations: + kubernetes.io/ingress.class: traefik + traefik.frontend.rule.type: PathPrefixStrip + # ingress.kubernetes.io/auth-type: "basic" + # ingress.kubernetes.io/auth-secret: "ingress-auth" +spec: + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: /moloch + backend: + serviceName: {{ template "moloch.name" . }}-viewer + servicePort: 8005 diff --git a/moloch/helm/templates/moloch-viewer-svc.yaml b/moloch/helm/templates/moloch-viewer-svc.yaml new file mode 100644 index 0000000..cbf8960 --- /dev/null +++ b/moloch/helm/templates/moloch-viewer-svc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "moloch.name" . }}-viewer + namespace: default + labels: + app: {{ template "moloch.name" . }} + role: viewer +spec: + ports: + - name: tcp-8005 + protocol: TCP + port: 8005 + targetPort: 8005 + selector: + app: {{ template "moloch.name" . }} + role: viewer + diff --git a/moloch/helm/templates/moloch-viewer.yaml b/moloch/helm/templates/moloch-viewer.yaml new file mode 100644 index 0000000..f8dcf20 --- /dev/null +++ b/moloch/helm/templates/moloch-viewer.yaml @@ -0,0 +1,67 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ template "moloch.fullname" . }}-viewer + labels: + app: {{ template "moloch.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + role: viewer +spec: + replicas: 1 + template: + metadata: + name: {{ template "moloch.name" . }}-viewer + labels: + app: {{ template "moloch.name" . }} + role: viewer + annotations: + networks: '[ + { "name": {{ .Values.networks.overlay | quote }} } + { "name": {{ .Values.networks.passive | quote }} } + ]' + spec: + containers: + - name: moloch-viewer + image: {{ .Values.images.moloch }} + imagePullPolicy: Always + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + env: + - name: ES_HOST + value: data-service + - name: NETWORK_INTERFACE + value: {{ .Values.networks.passive }} + - name: CLUSTER_PW + value: {{ .Values.molochConfig.env.clusterpw }} + - name: ADMIN_PW + value: {{ .Values.molochConfig.env.adminpw }} + - name: SENSOR + value: "false" + - name: VIEWER + value: "true" + ports: + - name: cport-8005 + containerPort: 8005 + resources: + requests: + cpu: {{ .Values.molochConfig.requests.cpu | quote }} + memory: {{ .Values.molochConfig.requests.memory | quote }} + limits: + cpu: {{ .Values.molochConfig.limits.cpu | quote }} + memory: {{ .Values.molochConfig.limits.memory | quote }} + volumeMounts: + - mountPath: /data/moloch/etc/config.ini + subPath: config.ini + name: moloch-config + volumes: + - name: moloch-config + configMap: + name: {{ template "moloch.fullname" . }}-config + restartPolicy: Always + dnsPolicy: ClusterFirst + nodeSelector: + nodetype: {{ .Values.nodeSelector.viewer }} diff --git a/moloch/helm/values.yaml b/moloch/helm/values.yaml new file mode 100644 index 0000000..9dff9d5 --- /dev/null +++ b/moloch/helm/values.yaml @@ -0,0 +1,41 @@ +# EDCOP Moloch Chart values +images: + moloch: miked235/moloch +networks: + # Overlay is the name of the default cni network + overlay: calico + passive: passive +volumes: + # Persistent data location on the host to store Moloch's data + logs: /bulk/EDCOP/moloch/logs + raw: /bulk/EDCOP/moloch/raw +nodeSelector: + worker: worker + viewer: master +ingress: + # Enter the FQDN of your host + host: physical.edcop.io +molochConfig: + # Nodes refers to the number of worker nodes you have + workerNodes: 3 + # These settings impact Moloch's overall performance + # https://github.com/aol/moloch/wiki/Settings#High_Performance_Settings + performance: + maxStreams: 1000000 + maxPacketsInQueue: 200000 + maxPackets: 10000 # Moloch writes a session record after this many + packetThreads: 5 + pcapWriteSize: 262143 # Buffer size when writing pcap files + tpacketv3Threads: 2 + env: + # Admin user for the web interface + adminpw: supersecretpw + clusterpw: anothersupersecretpw + encryptpw: randencryptpw # Recommanded to be random, never have to type it in + # Requests are set to accommodate limited resource VMs + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 4 + memory: 8G diff --git a/packetbeat/README.md b/packetbeat/README.md index c7ee6d5..e43aa10 100644 --- a/packetbeat/README.md +++ b/packetbeat/README.md @@ -18,13 +18,13 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing these values, make sure you use the full repository name. +By default, images are pulled from each application's respective official repository. If you're changing these values, make sure you use the full repository name and ensure your ELK stack versions match. ``` images: - packetbeat: edcop-master:5000/packetbeat - logstash: edcop-master:5000/logstash - redis: edcop-master:5000/redis + packetbeat: docker.elastic.co/beats/packetbeat:6.2.4 + logstash: docker.elastic.co/logstash/logstash:6.2.4 + redis: redis:4.0.9 ``` ## Networks @@ -61,9 +61,9 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` ## Packetbeat Configuration @@ -72,6 +72,9 @@ Currently, Packetbeat configuration settings only consist of limiting CPU/memory ``` packetbeatConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 4G @@ -90,6 +93,9 @@ logstashConfig: maxJvmHeap: 4g pipelineOutputWorkers: 2 pipelineBatchSize: 150 + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G @@ -101,6 +107,9 @@ Redis is also included in the Daemonset for the same reasons Logstash is. Curren ``` redisConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G diff --git a/stenographer/README.md b/stenographer/README.md index b0cc53d..6642e7a 100644 --- a/stenographer/README.md +++ b/stenographer/README.md @@ -19,11 +19,11 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing these values, make sure you use the full repository name. +By default, images are pulled from miked235/stenographer, which is a customized stenographer image based on centos7. ``` images: - stenographer: edcop-master:5000/stenographer + stenographer: miked235/stenographer ``` ## Networks @@ -71,9 +71,9 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` ## Stenographer Configuration @@ -96,6 +96,9 @@ You can set limits on Stenographer to ensure it doesn't use more CPU/memory spac ``` stenographerConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 4G diff --git a/stenographer/container/Dockerfile b/stenographer/container/Dockerfile index 419265b..ca5ea46 100644 --- a/stenographer/container/Dockerfile +++ b/stenographer/container/Dockerfile @@ -14,9 +14,9 @@ RUN go get github.com/google/stenographer && \ make -C stenotype && \ adduser --system --no-create-home stenographer && \ #sudo cp -v configs/limits.conf /etc/security/limits.d/stenographer.conf && \ - mkdir /etc/stenographer /etc/stenographer/certs /data /data/stenographer /data/stenographer/logs \ - /data/stenographer/index && \ - chown -R root:root /etc/stenographer/certs + mkdir /etc/stenographer /etc/stenographer/certs /data /data/stenographer /data/stenographer/logs /data/stenographer/index && \ + chown -R root:root /etc/stenographer/certs && \ + chown -R stenographer:stenographer /data/stenographer # Copy over default config COPY config /etc/stenographer/config @@ -34,5 +34,8 @@ RUN chmod 644 /etc/stenographer/config && \ # Run as user stenographer USER stenographer +# Make sure the port matches the config +EXPOSE 1234 + CMD ["stenographer", "-syslog=false"] diff --git a/stenographer/container/config b/stenographer/container/config index 480ad9a..1b029e9 100644 --- a/stenographer/container/config +++ b/stenographer/container/config @@ -6,7 +6,7 @@ } ] , "StenotypePath": "/usr/bin/stenotype" - , "Interface": "net0" + , "Interface": "eth0" , "Port": 1234 , "Host": "127.0.0.1" , "Flags": ["-v"] diff --git a/stenographer/helm/templates/stenographer-etc-config.yaml b/stenographer/helm/templates/stenographer-etc-config.yaml index dc6f2b7..7690067 100644 --- a/stenographer/helm/templates/stenographer-etc-config.yaml +++ b/stenographer/helm/templates/stenographer-etc-config.yaml @@ -14,7 +14,7 @@ data: ] , "StenotypePath": "/usr/bin/stenotype" , "Interface": "net0" - , "Port": 1234 + , "Port": 9005 , "Host": "127.0.0.1" , "Flags": ["-v"] , "CertPath": "/etc/stenographer/certs" diff --git a/stenographer/helm/values.yaml b/stenographer/helm/values.yaml index 5d0e06c..e2f3f12 100644 --- a/stenographer/helm/values.yaml +++ b/stenographer/helm/values.yaml @@ -1,6 +1,6 @@ # EDCOP Stenographer Chart values images: - stenographer: edcop-master:5000/stenographer + stenographer: miked235/stenographer networks: # Overlay is the name of the default cni network overlay: calico diff --git a/suricata/README.md b/suricata/README.md index 000e307..575f0c6 100644 --- a/suricata/README.md +++ b/suricata/README.md @@ -23,7 +23,7 @@ Please share any bugs or features requests via GitHub issues. ## Image Repository -By default, images are pulled from *edcop-master:5000* which is presumed to be hosted on the master node. If you're changing this value, make sure you use the full repository name. +By default, images are pulled from EDCOP's official repository as well as the official repositories for Logstash and Redis. ``` images: @@ -37,13 +37,6 @@ images: Suricata uses 2 or 3 interfaces depending on whether it is in passive or inline mode. If you choose passive mode, net2 will be ignored and net1 will be the name of the passive interface. By default, these interfaces are named *calico*, *passive*, *inline-1*, and *inline-2*. -``` -networks: - overlay: calico - net1: passive - net2: -``` - ``` networks: overlay: calico @@ -74,9 +67,9 @@ To find out what labels your hosts have, please use the following: ``` # kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS -master Ready master 1d v1.9.1 ...,nodetype=master -minion-1 Ready 1d v1.9.1 ...,nodetype=minion -minion-2 Ready 1d v1.9.1 ...,nodetype=minion +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion ``` ## Deployment Options @@ -148,6 +141,9 @@ You can set limits on Suricata to ensure it doesn't use more CPU/memory space th ``` suricataConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 4G @@ -166,6 +162,9 @@ logstashConfig: maxJvmHeap: 4g pipelineOutputWorkers: 2 pipelineBatchSize: 150 + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G @@ -177,6 +176,9 @@ Redis is also included in the Daemonset (except for external mode) for the same ``` redisConfig: + requests: + cpu: 100m + memory: 64Mi limits: cpu: 2 memory: 8G diff --git a/wireshark/README.md b/wireshark/README.md new file mode 100644 index 0000000..229d86c --- /dev/null +++ b/wireshark/README.md @@ -0,0 +1,130 @@ +# EDCOP Wireshark Guide + +Table of Contents +----------------- + +* [Configuration Guide](#configuration-guide) + * [Image Repository](#image-repository) + * [Networks](#networks) + * [Node Selector](#node-selector) + * [Wireshark Configuration](#wireshark-configuration) + * [Resource Limits](#resource-limits) + * [Logstash Configuration](#logstash-configuration) + * [Redis Configuration](#redis-configuration) + +# Configuration Guide + +Within this configuration guide, you will find instructions for modifying Wireshark's helm chart. All changes should be made in the *values.yaml* file. +Please share any bugs or features requests via GitHub issues. + +## Image Repository + +By default, images are pulled from official images hosted on Docker's hub alongside our custom Wireshark image. If you're changing these values, make sure you use the full repository name. + +``` +images: + wireshark: miked235/wireshark + logstash: docker.elastic.co/logstash/logstash:6.2.4 + redis: redis:4.0.9 + filebeat: docker.elastic.co/beats/filebeat:6.2.4 +``` + +## Networks + +Wireshark only uses 2 interfaces because it can only be deployed in passive mode. By default, these networks are named *calico* and *passive*. + +``` +networks: + overlay: calico + passive: passive +``` + +To find the names of your networks, use the following command: + +``` +# kubectl get networks +NAME AGE +calico 1d +passive 1d +inline-1 1d +inline-2 1d +``` + +## Node Selector + +This value tells Kubernetes which hosts the daemonset should be deployed to by using labels given to the hosts. Hosts without the defined label will not receive pods. + +``` +nodeSelector: + nodetype: worker +``` + +To find out what labels your hosts have, please use the following: +``` +# kubectl get nodes --show-labels +NAME STATUS ROLES AGE VERSION LABELS +master Ready master 1d v1.10.0 ...,nodetype=master +minion-1 Ready 1d v1.10.0 ...,nodetype=minion +minion-2 Ready 1d v1.10.0 ...,nodetype=minion +``` + +## Wireshark Configuration + +Wireshark is used as a passive network inspection tool, so no advanced configuration is required for accepting traffic. Wireshark is intended to be a troubleshooting only tool and should not be used in place of an IDS/FPCAP solution. + +You can specify what kind of traffic you're interested in by giving Wireshark various commandline arguments. You can refer to https://www.wireshark.org/docs/man-pages/tshark.html for information on custom command flags: + +``` +wiresharkConfig: + env: + args: "-j ip" +``` + +### Resource Limits + +You can set limits on Wireshark to ensure it doesn't use more CPU/memory space than necessary: + +``` +wiresharkConfig: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 4G +``` + +## Logstash Configuration + +Logstash is currently included in the Daemonset to streamline the rules required for the data it ingests. Having one Logstash instance per node would clutter rules and cause congestion with log filtering, which would harm our events/second speed. This instance will only deal with Wireshark's logs and doesn't need complicated filters to figure out which tool the logs came from. +Please make sure to read the [Logstash Performance Tuning Guide](https://www.elastic.co/guide/en/logstash/current/performance-troubleshooting.html) for a better understanding of managing Logstash's resources. + +``` +logstashConfig: + threads: 2 + batchCount: 250 + initialJvmHeap: 4g + maxJvmHeap: 4g + pipelineOutputWorkers: 2 + pipelineBatchSize: 150 + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 8G +``` + +## Redis Configuration + +Redis is also included in the Daemonset for the same reasons Logstash is. Currently, you can only limit the resources of Redis in this section, but in the future we would like to add configmaps for tuning purposes. + +``` +redisConfig: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 8G +``` diff --git a/wireshark/helm/Chart.yaml b/wireshark/helm/Chart.yaml new file mode 100644 index 0000000..33e9c0e --- /dev/null +++ b/wireshark/helm/Chart.yaml @@ -0,0 +1,9 @@ +piVersion: v1 +name: wireshark +home: https://github.com/sealingtech/EDCOP +version: 0.1.0 +description: EDCOP Wireshark Chart +details: + This Chart provides a Wireshark daemonset for use with the EDCOP project. +icon: https://www.wireshark.org/assets/theme-2015/images/wireshark_logo@2x.png + diff --git a/wireshark/helm/templates/_helpers.tpl b/wireshark/helm/templates/_helpers.tpl new file mode 100644 index 0000000..80bebed --- /dev/null +++ b/wireshark/helm/templates/_helpers.tpl @@ -0,0 +1,17 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "wireshark.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "wireshark.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + diff --git a/wireshark/helm/templates/filebeat-wireshark-confg.yaml b/wireshark/helm/templates/filebeat-wireshark-confg.yaml new file mode 100644 index 0000000..326601b --- /dev/null +++ b/wireshark/helm/templates/filebeat-wireshark-confg.yaml @@ -0,0 +1,875 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "wireshark.fullname" . }}-filebeat-config +data: + filebeat.yml: | + ######################## Filebeat Configuration ############################ + + # This file is a full configuration example documenting all non-deprecated + # options in comments. For a shorter configuration example, that contains only + # the most common options, please see filebeat.yml in the same directory. + # + # You can find the full configuration reference here: + # https://www.elastic.co/guide/en/beats/filebeat/index.html + + + #========================== Modules configuration ============================ + filebeat.modules: + + + #=========================== Filebeat prospectors ============================= + + # List of prospectors to fetch data. + filebeat.prospectors: + # Each - is a prospector. Most options can be set at the prospector level, so + # you can use different prospectors for various configurations. + # Below are the prospector specific configurations. + + # Type of the files. Based on this the way the file is read is decided. + # The different types cannot be mixed in one prospector + + #------------------------------ Log prospector -------------------------------- + - type: log + paths: + - /wireshark/packets.json + document_type: "pcap_file" + json.keys_under_root: true + tags: ["wireshark"] + + ### JSON configuration + + # Decode JSON options. Enable this if your logs are structured in JSON. + # JSON key on which to apply the line filtering and multiline settings. This key + # must be top level and its value must be string, otherwise it is ignored. If + # no text key is defined, the line filtering and multiline features cannot be used. + #json.message_key: + + # If keys_under_root and this setting are enabled, then the values from the decoded + # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) + # in case of conflicts. + #json.overwrite_keys: false + + # If this setting is enabled, Filebeat adds a "error.message" and "error.key: json" key in case of JSON + # unmarshaling errors or when a text key is defined in the configuration but cannot + # be used. + #json.add_error_key: true + + ### Multiline options + + # Mutiline can be used for log messages spanning multiple lines. This is common + # for Java Stack Traces or C-Line Continuation + + # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [ + #multiline.pattern: ^\[ + + # Defines if the pattern set under pattern should be negated or not. Default is false. + #multiline.negate: false + + # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern + # that was (not) matched before or after or as long as a pattern is not matched based on negate. + # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash + #multiline.match: after + + # The maximum number of lines that are combined to one event. + # In case there are more the max_lines the additional lines are discarded. + # Default is 500 + #multiline.max_lines: 500 + + # After the defined timeout, an multiline event is sent even if no new pattern was found to start a new event + # Default is 5s. + #multiline.timeout: 5s + + # Setting tail_files to true means filebeat starts reading new files at the end + # instead of the beginning. If this is used in combination with log rotation + # this can mean that the first entries of a new file are skipped. + #tail_files: false + + # The Ingest Node pipeline ID associated with this prospector. If this is set, it + # overwrites the pipeline option from the Elasticsearch output. + #pipeline: + + # If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the + # original for harvesting but will report the symlink name as source. + #symlinks: false + + # Backoff values define how aggressively filebeat crawls new files for updates + # The default values can be used in most cases. Backoff defines how long it is waited + # to check a file again after EOF is reached. Default is 1s which means the file + # is checked every second if new lines were added. This leads to a near real time crawling. + # Every time a new line appears, backoff is reset to the initial value. + #backoff: 1s + + # Max backoff defines what the maximum backoff time is. After having backed off multiple times + # from checking the files, the waiting time will never exceed max_backoff independent of the + # backoff factor. Having it set to 10s means in the worst case a new line can be added to a log + # file after having backed off multiple times, it takes a maximum of 10s to read the new line + #max_backoff: 10s + + # The backoff factor defines how fast the algorithm backs off. The bigger the backoff factor, + # the faster the max_backoff value is reached. If this value is set to 1, no backoff will happen. + # The backoff value will be multiplied each time with the backoff_factor until max_backoff is reached + #backoff_factor: 2 + + # Max number of harvesters that are started in parallel. + # Default is 0 which means unlimited + #harvester_limit: 0 + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #close_inactive: 5m + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close_renamed: false + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + #close_removed: true + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close_eof: false + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + #clean_inactive: + # Removes the state for file which cannot be found on disk anymore immediately + #clean_removed: true + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close_timeout: 0 + + # Defines if prospectors is enabled + #enabled: true + + #----------------------------- Stdin prospector ------------------------------- + # Configuration to use stdin input + #- type: stdin + + #------------------------- Redis slowlog prospector --------------------------- + # Experimental: Config options for the redis slow log prospector + #- type: redis + #hosts: ["localhost:6379"] + #username: + #password: + #enabled: true + #scan_frequency: 10s + + # Timeout after which time the prospector should return an error + #timeout: 1s + + # Network type to be used for redis connection. Default: tcp + #network: tcp + + # Max number of concurrent connections. Default: 10 + #maxconn: 10 + + # Redis AUTH password. Empty by default. + #password: foobared + + #========================= Filebeat global options ============================ + + # Event count spool threshold - forces network flush if exceeded + #filebeat.spool_size: 2048 + + # Enable async publisher pipeline in filebeat (Experimental!) + #filebeat.publish_async: false + + # Defines how often the spooler is flushed. After idle_timeout the spooler is + # Flush even though spool_size is not reached. + #filebeat.idle_timeout: 5s + + # Name of the registry file. If a relative path is used, it is considered relative to the + # data path. + #filebeat.registry_file: ${path.data}/registry + + # + # These config files must have the full filebeat config part inside, but only + # the prospector part is processed. All global options like spool_size are ignored. + # The config_dir MUST point to a different directory then where the main filebeat config file is in. + #filebeat.config_dir: + + # How long filebeat waits on shutdown for the publisher to finish. + # Default is 0, not waiting. + #filebeat.shutdown_timeout: 0 + + # Enable filebeat config reloading + #filebeat.config.prospectors: + #enabled: false + #path: configs/*.yml + #reload.enabled: true + #reload.period: 10s + + #================================ General ====================================== + + # The name of the shipper that publishes the network data. It can be used to group + # all the transactions sent by a single shipper in the web interface. + # If this options is not defined, the hostname is used. + #name: + + # The tags of the shipper are included in their own field with each + # transaction published. Tags make it easy to group servers by different + # logical properties. + #tags: ["service-X", "web-tier"] + + # Optional fields that you can specify to add additional information to the + # output. Fields can be scalar values, arrays, dictionaries, or any nested + # combination of these. + #fields: + # env: staging + + # If this option is set to true, the custom fields are stored as top-level + # fields in the output document instead of being grouped under a fields + # sub-dictionary. Default is false. + #fields_under_root: false + + # Internal queue size for single events in processing pipeline + #queue_size: 1000 + + # The internal queue size for bulk events in the processing pipeline. + # Do not modify this value. + #bulk_queue_size: 0 + + # Sets the maximum number of CPUs that can be executing simultaneously. The + # default is the number of logical CPUs available in the system. + #max_procs: + + #================================ Processors =================================== + + # Processors are used to reduce the number of fields in the exported event or to + # enhance the event with external metadata. This section defines a list of + # processors that are applied one by one and the first one receives the initial + # event: + # + # event -> filter1 -> event1 -> filter2 ->event2 ... + # + # The supported processors are drop_fields, drop_event, include_fields, and + # add_cloud_metadata. + # + # For example, you can use the following processors to keep the fields that + # contain CPU load percentages, but remove the fields that contain CPU ticks + # values: + # + #processors: + #- include_fields: + # fields: ["cpu"] + #- drop_fields: + # fields: ["cpu.user", "cpu.system"] + # + # The following example drops the events that have the HTTP response code 200: + + # https://www.elastic.co/blog/analyzing-network-packets-with-wireshark-elasticsearch-and-kibana + processors: + - drop_event: + when: + equals: + index._type: "pcap_file" + + # The following example enriches each event with metadata from the cloud + # provider about the host machine. It works on EC2, GCE, DigitalOcean, + # Tencent Cloud, and Alibaba Cloud. + # + #processors: + #- add_cloud_metadata: ~ + # + # The following example enriches each event with the machine's local time zone + # offset from UTC. + # + #processors: + #- add_locale: + # format: offset + # + # The following example enriches each event with docker metadata, it matches + # given fields to an existing container id and adds info from that container: + # + #processors: + #- add_docker_metadata: + # match_fields: ["system.process.cgroup.id"] + # host: "unix:///var/run/docker.sock" + # # To connect to Docker over TLS you must specify a client and CA certificate. + # #ssl: + # # certificate_authority: "/etc/pki/root/ca.pem" + # # certificate: "/etc/pki/client/cert.pem" + # # key: "/etc/pki/client/cert.key" + # + + #================================ Outputs ====================================== + + # Configure what outputs to use when sending the data collected by the beat. + # Multiple outputs may be used. + + #-------------------------- Elasticsearch output ------------------------------- + #output.elasticsearch: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the url with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Number of workers per Elasticsearch host. + #worker: 1 + + # Optional index name. The default is "filebeat" plus date + # and generates [filebeat-]YYYY.MM.DD keys. + #index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}" + + # Optional ingest node pipeline. By default no pipeline will be used. + #pipeline: "" + + # Optional HTTP Path + #path: "/elasticsearch" + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # Configure http request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # The number of seconds to wait for new events between two bulk API index requests. + # If `bulk_max_size` is reached before this interval expires, addition bulk index + # requests are made. + #flush_interval: 1s + + # Use SSL settings for HTTPS. Default is true. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + + #----------------------------- Logstash output --------------------------------- + #output.logstash: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Optional load balance the events between the Logstash hosts + #loadbalance: true + + # Number of batches to be send asynchronously to logstash while processing + # new batches. + #pipelining: 0 + + # Optional index name. The default index name is set to name of the beat + # in all lowercase. + #index: 'filebeat' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + #------------------------------- Kafka output ---------------------------------- + #output.kafka: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. The setting can be a format string + # using any event field. To set the topic from document type use `%{[type]}`. + #topic: beats + + # The Kafka event key setting. Use format string to create unique event key. + # By default no event key will be generated. + #key: '' + + # The Kafka event partitioning strategy. Default hashing strategy is `hash` + # using the `output.kafka.key` setting or randomly distributes events if + # `output.kafka.key` is not configured. + #partition.hash: + # If enabled, events will only be published to partitions with reachable + # leaders. Default is false. + #reachable_only: false + + # Configure alternative event field names used to compute the hash value. + # If empty `output.kafka.key` setting will be used. + # Default value is empty list. + #hash: [] + + # Authentication details. Password is required if username is set. + #username: '' + #password: '' + + # Kafka version filebeat is assumed to run against. Defaults to the oldest + # supported stable version (currently version 0.8.2.0) + #version: 0.8.2 + + # Metadata update configuration. Metadata do contain leader information + # deciding which broker to use when publishing. + #metadata: + # Max metadata request retry attempts when cluster is in middle of leader + # election. Defaults to 3 retries. + #retry.max: 3 + + # Waiting time between retries during leader elections. Default is 250ms. + #retry.backoff: 250ms + + # Refresh metadata interval. Defaults to every 10 minutes. + #refresh_frequency: 10m + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The number of seconds to wait for new events between two producer API calls. + #flush_interval: 1s + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + #------------------------------- Redis output ---------------------------------- + output.redis: + # Boolean flag to enable or disable the output module. + enabled: true + + # The list of Redis servers to connect to. If load balancing is enabled, the + # events are distributed to the servers in the list. If one server becomes + # unreachable, the events are distributed to the reachable servers only. + hosts: ["localhost"] + + # The Redis port to use if hosts does not contain a port number. The default + # is 6379. + port: 6379 + + # The name of the Redis list or channel the events are published to. The + # default is filebeat. + key: wireshark + + # The password to authenticate with. The default is no authentication. + #password: + + # The Redis database number where the events are published. The default is 0. + db: 0 + + # The Redis data type to use for publishing events. If the data type is list, + # the Redis RPUSH command is used. If the data type is channel, the Redis + # PUBLISH command is used. The default value is list. + datatype: list + + # The number of workers to use for each host configured to publish events to + # Redis. Use this setting along with the loadbalance option. For example, if + # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each + # host). + #worker: 1 + + # If set to true and multiple hosts or workers are configured, the output + # plugin load balances published events onto all Redis hosts. If set to false, + # the output plugin sends all events to only one host (determined at random) + # and will switch to another host if the currently selected one becomes + # unreachable. The default value is true. + #loadbalance: true + + # The Redis connection timeout in seconds. The default is 5 seconds. + # timeout: 5s + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Redis request or pipeline. + # The default is 2048. + #bulk_max_size: 2048 + + # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The + # value must be a URL with a scheme of socks5://. + #proxy_url: + + # This option determines whether Redis hostnames are resolved locally when + # using a proxy. The default value is false, which means that name resolution + # occurs on the proxy server. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + + #------------------------------- File output ----------------------------------- + #output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/filebeat" + + # Name of the generated files. The default is `filebeat` and it generates + # files: `filebeat`, `filebeat.1`, `filebeat.2`, etc. + #filename: filebeat + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every filebeat restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + + #----------------------------- Console output --------------------------------- + #output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Pretty print json event + #pretty: false + + #================================= Paths ====================================== + + # The home path for the filebeat installation. This is the default base path + # for all other path settings and for miscellaneous files that come with the + # distribution (for example, the sample dashboards). + # If not set by a CLI flag or in the configuration file, the default for the + # home path is the location of the binary. + #path.home: + + # The configuration path for the filebeat installation. This is the default + # base path for configuration files, including the main YAML configuration file + # and the Elasticsearch template file. If not set by a CLI flag or in the + # configuration file, the default for the configuration path is the home path. + #path.config: ${path.home} + + # The data path for the filebeat installation. This is the default base path + # for all the files in which filebeat needs to store its data. If not set by a + # CLI flag or in the configuration file, the default for the data path is a data + # subdirectory inside the home path. + #path.data: ${path.home}/data + + # The logs path for a filebeat installation. This is the default location for + # the Beat's log files. If not set by a CLI flag or in the configuration file, + # the default for the logs path is a logs subdirectory inside the home path. + #path.logs: ${path.home}/logs + + #============================== Dashboards ===================================== + # These settings control loading the sample dashboards to the Kibana index. Loading + # the dashboards is disabled by default and can be enabled either by setting the + # options here, or by using the `-setup` CLI flag. + #setup.dashboards.enabled: false + + # The URL from where to download the dashboards archive. By default this URL + # has a value which is computed based on the Beat name and version. For released + # versions, this URL points to the dashboard archive on the artifacts.elastic.co + # website. + #setup.dashboards.url: + + # The directory from where to read the dashboards. It is used instead of the URL + # when it has a value. + #setup.dashboards.directory: + + # The file archive (zip file) from where to read the dashboards. It is used instead + # of the URL when it has a value. + #setup.dashboards.file: + + # If this option is enabled, the snapshot URL is used instead of the default URL. + #setup.dashboards.snapshot: false + + # The URL from where to download the snapshot version of the dashboards. By default + # this has a value which is computed based on the Beat name and version. + #setup.dashboards.snapshot_url + + # In case the archive contains the dashboards from multiple Beats, this lets you + # select which one to load. You can load all the dashboards in the archive by + # setting this to the empty string. + #setup.dashboards.beat: filebeat + + # The name of the Kibana index to use for setting the configuration. Default is ".kibana" + #setup.dashboards.kibana_index: .kibana + + # The Elasticsearch index name. This overwrites the index name defined in the + # dashboards and index pattern. Example: testbeat-* + #setup.dashboards.index: + + #============================== Template ===================================== + + # A template is used to set the mapping in Elasticsearch + # By default template loading is enabled and the template is loaded. + # These settings can be adjusted to load your own template or overwrite existing ones. + + # Set to false to disable template loading. + #setup.template.enabled: true + + # Template name. By default the template name is filebeat. + # The version of the beat will always be appended to the given name + # so the final name is filebeat-%{[beat.version]}. + #setup.template.name: "filebeat" + + # Path to fields.yml file to generate the template + #setup.template.fields: "${path.config}/fields.yml" + + # Overwrite existing template + #setup.template.overwrite: false + + + #================================ HTTP Endpoint ====================================== + # Each beat can expose internal data points through a http endpoint. For security + # reason the endpoint is disabled by default. This feature is currently in beta. + + # Defines if http endpoint is enabled + #http.enabled: false + + # Host to expose the http endpoint to. It is recommended to use only localhost. + #http.host: localhost + + # Port on which the http endpoint is exposed. Default is 5066 + #http.port: 5066 + + #================================ Logging ====================================== + # There are three options for the log output: syslog, file, stderr. + # Under Windows systems, the log files are per default sent to the file output, + # under all other system per default to syslog. + + # Sets log level. The default log level is info. + # Available log levels are: critical, error, warning, info, debug + #logging.level: info + + # Enable debug output for selected components. To enable all selectors use ["*"] + # Other available selectors are "beat", "publish", "service" + # Multiple selectors can be chained. + #logging.selectors: [ ] + + # Send all logging output to syslog. The default is false. + #logging.to_syslog: true + + # If enabled, filebeat periodically logs its internal metrics that have changed + # in the last period. For each metric that changed, the delta from the value at + # the beginning of the period is logged. Also, the total values for + # all non-zero internal metrics are logged on shutdown. The default is true. + #logging.metrics.enabled: true + + # The period after which to log the internal metrics. The default is 30s. + #logging.metrics.period: 30s + + # Logging to rotating files files. Set logging.to_files to false to disable logging to + # files. + #logging.to_files: true + #logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/filebeat + + # The name of the files where the logs are written to. + #name: filebeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 diff --git a/wireshark/helm/templates/logstash-wireshark-config.yaml b/wireshark/helm/templates/logstash-wireshark-config.yaml new file mode 100644 index 0000000..4f0fa16 --- /dev/null +++ b/wireshark/helm/templates/logstash-wireshark-config.yaml @@ -0,0 +1,209 @@ +piVersion: v1 +kind: ConfigMap +metadata: + name: logstash-{{ template "wireshark.fullname" . }}-config +data: + jvm.options: | + ## JVM configuration + # Xms represents the initial size of total heap space + # Xmx represents the maximum size of total heap space + -Xms{{ .Values.logstashConfig.initialJvmHeap }} + -Xmx{{ .Values.logstashConfig.maxJvmHeap }} + ################################################################ + ## Expert settings + ################################################################ + ## + ## All settings below this section are considered + ## expert settings. Don't tamper with them unless + ## you understand what you are doing + ## + ################################################################ + ## GC configuration + -XX:+UseParNewGC + -XX:+UseConcMarkSweepGC + -XX:CMSInitiatingOccupancyFraction=75 + -XX:+UseCMSInitiatingOccupancyOnly + ## optimizations + # disable calls to System#gc + -XX:+DisableExplicitGC + ## Locale + # Set the locale language + #-Duser.language=en + # Set the locale country + #-Duser.country=US + # Set the locale variant, if any + #-Duser.variant= + ## basic + # set the I/O temp directory + #-Djava.io.tmpdir=$HOME + # set to headless, just in case + -Djava.awt.headless=true + # ensure UTF-8 encoding by default (e.g. filenames) + -Dfile.encoding=UTF-8 + # use our provided JNA always versus the system one + #-Djna.nosys=true + # Turn on JRuby invokedynamic + # -Djruby.compile.invokedynamic=true + ## heap dumps + # generate a heap dump when an allocation from the Java heap fails + # heap dumps are created in the working directory of the JVM + -XX:+HeapDumpOnOutOfMemoryError + # specify an alternative path for heap dumps + # ensure the directory exists and has sufficient space + #-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof + ## GC logging + #-XX:+PrintGCDetails + #-XX:+PrintGCTimeStamps + #-XX:+PrintGCDateStamps + #-XX:+PrintClassHistogram + #-XX:+PrintTenuringDistribution + #-XX:+PrintGCApplicationStoppedTime + # log GC status to a file with time stamps + # ensure the directory exists + #-Xloggc:${LS_GC_LOG_FILE} + # Entropy source for randomness + -Djava.security.egd=file:/dev/urandom + log4j2.properties : | + status = error + name = LogstashPropertiesConfig + appender.console.type = Console + appender.console.name = plain_console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + appender.json_console.type = Console + appender.json_console.name = json_console + appender.json_console.layout.type = JSONLayout + appender.json_console.layout.compact = true + appender.json_console.layout.eventEol = true + rootLogger.level = ${sys:ls.log.level} + rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console + logstash.yml : | + http.host: "0.0.0.0" + path.config: /usr/share/logstash/pipeline + xpack.monitoring.enabled: true + xpack.monitoring.elasticsearch.url: http://data-service:9200 + xpack.monitoring.elasticsearch.username: logstash_system + xpack.monitoring.elasticsearch.password: changeme + pipelines.yml : | + # List of pipelines to be loaded by Logstash + # + # This document must be a list of dictionaries/hashes, where the keys/values are pipeline settings. + # Default values for ommitted settings are read from the `logstash.yml` file. + # When declaring multiple pipelines, each MUST have its own `pipeline.id`. + # + # Example of two pipelines: + # + # - pipeline.id: test + # pipeline.workers: 1 + # pipeline.batch.size: 1 + # config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }" + # - pipeline.id: another_test + # queue.type: persisted + # path.config: "/tmp/logstash/*.config" + # + # Available options: + # + # # name of the pipeline + # pipeline.id: mylogs + # + # # The configuration string to be used by this pipeline + # config.string: "input { generator {} } filter { sleep { time => 1 } } output { stdout { codec => dots } }" + # + # # The path from where to read the configuration text + # path.config: "/etc/conf.d/logstash/myconfig.cfg" + # + # # How many worker threads execute the Filters+Outputs stage of the pipeline + # pipeline.workers: 1 (actually defaults to number of CPUs) + pipeline.workers: {{ .Values.logstashConfig.pipelineWorkers }} + # + # # How many events to retrieve from inputs before sending to filters+workers + pipeline.batch.size: {{ .Values.logstashConfig.pipelineBatchSize }} + # + # # How long to wait before dispatching an undersized batch to filters+workers + # pipeline.batch.delay: 5 + # + # # How many workers should be used per output plugin instance + pipeline.output.workers: {{ .Values.logstashConfig.pipelineOutputWorkers }} + # + # # Internal queuing model, "memory" for legacy in-memory based queuing and + # # "persisted" for disk-based acked queueing. Defaults is memory + # queue.type: memory + # + # # If using queue.type: persisted, the page data files size. The queue data consists of + # # append-only data files separated into pages. Default is 250mb + # queue.page_capacity: 250mb + # + # # If using queue.type: persisted, the maximum number of unread events in the queue. + # # Default is 0 (unlimited) + # queue.max_events: 0 + # + # # If using queue.type: persisted, the total capacity of the queue in number of bytes. + # # Default is 1024mb or 1gb + # queue.max_bytes: 1024mb + # + # # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint + # # Default is 1024, 0 for unlimited + # queue.checkpoint.acks: 1024 + # + # # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint + # # Default is 1024, 0 for unlimited + # queue.checkpoint.writes: 1024 + # + # # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page + # # Default is 1000, 0 for no periodic checkpoint. + # queue.checkpoint.interval: 1000 + # + # # Enable Dead Letter Queueing for this pipeline. + # dead_letter_queue.enable: false + # + # If using dead_letter_queue.enable: true, the maximum size of dead letter queue for this pipeline. Entries + # will be dropped if they would increase the size of the dead letter queue beyond this setting. + # Default is 1024mb + # dead_letter_queue.max_bytes: 1024mb + # + # If using dead_letter_queue.enable: true, the directory path where the data files will be stored. + # Default is path.data/dead_letter_queue + # + # path.dead_letter_queue: + startup.options : | + ################################################################################ + # These settings are ONLY used by $LS_HOME/bin/system-install to create a custom + # startup script for Logstash and is not used by Logstash itself. It should + # automagically use the init system (systemd, upstart, sysv, etc.) that your + # Linux distribution uses. + # + # After changing anything here, you need to re-run $LS_HOME/bin/system-install + # as root to push the changes to the init script. + ################################################################################ + # Override Java location + #JAVACMD=/usr/bin/java + # Set a home directory + LS_HOME=/usr/share/logstash + # logstash settings directory, the path which contains logstash.yml + LS_SETTINGS_DIR="${LS_HOME}/config" + # Arguments to pass to logstash + LS_OPTS="--path.settings ${LS_SETTINGS_DIR}" + # Arguments to pass to java + LS_JAVA_OPTS="" + # pidfiles aren't used the same way for upstart and systemd; this is for sysv users. + LS_PIDFILE=/var/run/logstash.pid + # user and group id to be invoked as + LS_USER=logstash + LS_GROUP=logstash + # Enable GC logging by uncommenting the appropriate lines in the GC logging + # section in jvm.options + LS_GC_LOG_FILE=/var/log/logstash/gc.log + # Open file limit + LS_OPEN_FILES=16384 + # Nice level + LS_NICE=19 + # Change these to have the init script named and described differently + # This is useful when running multiple instances of Logstash on the same + # physical box or vm + SERVICE_NAME="logstash" + SERVICE_DESCRIPTION="logstash" + # If you need to run a command or script before launching Logstash, put it + # between the lines beginning with `read` and `EOM`, and uncomment those lines. + ### + ## read -r -d '' PRESTART << EOM + ## EOM diff --git a/wireshark/helm/templates/logstash-wireshark-pipeline.yaml b/wireshark/helm/templates/logstash-wireshark-pipeline.yaml new file mode 100644 index 0000000..789ae88 --- /dev/null +++ b/wireshark/helm/templates/logstash-wireshark-pipeline.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: logstash-{{ template "wireshark.fullname" . }}-pipeline +data: + wireshark.conf: | + input { + redis { + host => "localhost" + key => "wireshark" + data_type => "list" + codec => json + batch_count => {{ .Values.logstashConfig.batchCount }} + threads => {{ .Values.logstashConfig.threads }} + } + } + + filter { + # Drop Elasticsearch Bulk API control lines + if ([message] =~ "{\"index") { + drop {} + } + + json { + source => "message" + remove_field => "message" + } + + # Extract innermost network protocol + grok { + match => { + "[layers][frame][frame_frame_protocols]" => "%{WORD:protocol}$" + } + } + + date { + match => [ "timestamp", "UNIX_MS" ] + } + } + + output { + elasticsearch { + hosts => "data-service:9200" + manage_template => false + index => "wireshark-%{+YYYY.MM.dd}" + document_type => "pcap_file" + codec => json + #user => logstash_internal + #password => changeme + } + } diff --git a/wireshark/helm/templates/wireshark-daemonset.yaml b/wireshark/helm/templates/wireshark-daemonset.yaml new file mode 100644 index 0000000..0e5a39e --- /dev/null +++ b/wireshark/helm/templates/wireshark-daemonset.yaml @@ -0,0 +1,128 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: {{ template "wireshark.fullname" . }} + labels: + app: {{ template "wireshark.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + matchLabels: + app: {{ template "wireshark.name" . }} + template: + metadata: + name: {{ template "wireshark.name" . }} + labels: + app: {{ template "wireshark.name" . }} + release: {{ .Release.Name }} + annotations: + networks: '[ + { "name": "{{ .Values.networks.overlay }}" }, + { "name": "{{ .Values.networks.passive }}" } + ]' + spec: + containers: + - name: wireshark + image: {{ .Values.images.wireshark }} + imagePullPolicy: Always + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + env: + - name: NETWORK_INTERFACE + value: net0 + - name: ARGS + value: "{{ .Values.wiresharkConfig.env.args }}" + resources: + requests: + cpu: "{{ .Values.wiresharkConfig.requests.cpu }}" + memory: "{{ .Values.wiresharkConfig.requests.memory }}" + limits: + cpu: "{{ .Values.wiresharkConfig.limits.cpu }}" + memory: "{{ .Values.wiresharkConfig.limits.memory }}" + volumeMounts: + - mountPath: /data/wireshark + name: wireshark-packets + + - name: redis + image: {{ .Values.images.redis }} + imagePullPolicy: Always + ports: + - name: cport-6379 + containerPort: 6379 + resources: + requests: + cpu: "{{ .Values.redisConfig.requests.cpu }}" + memory: "{{ .Values.redisConfig.requests.memory }}" + limits: + cpu: "{{ .Values.redisConfig.limits.cpu }}" + memory: "{{ .Values.redisConfig.limits.memory }}" + + - name: logstash + image: {{ .Values.images.logstash }} + imagePullPolicy: Always + ports: + - name: cport-5044 + containerPort: 5044 + resources: + requests: + cpu: "{{ .Values.logstashConfig.requests.cpu }}" + memory: "{{ .Values.logstashConfig.requests.memory }}" + limits: + cpu: "{{ .Values.logstashConfig.limits.cpu }}" + memory: "{{ .Values.logstashConfig.limits.memory }}" + volumeMounts: + - mountPath: /usr/share/logstash/pipeline/ + name: logstash-wireshark-pipeline + - mountPath: /usr/share/logstash/config/ + name: logstash-wireshark-config + + - name: filebeat + image: {{ .Values.images.filebeat }} + imagePullPolicy: Always + volumeMounts: + - mountPath: /wireshark/ + name: wireshark-packets + - mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + name: filebeat-wireshark-config + + initContainers: + - name: wireshark-index + image: centos + command: + - chmod + - "+x" + - "/scripts/wireshark-index.sh" + - "&&" + - "/scripts/wireshark-index.sh" + volumeMounts: + - name: wireshark-index-script + mountPath: /scripts/wireshark-index.sh + subPath: wireshark-index.sh + + volumes: + - name: wireshark-packets + emptyDir: + medium: "Memory" + - name: filebeat-wireshark-config + configMap: + name: {{ template "wireshark.fullname" . }}-filebeat-config + - name: logstash-wireshark-config + configMap: + name: logstash-{{ template "wireshark.fullname" . }}-config + - name: logstash-wireshark-pipeline + configMap: + name: logstash-{{ template "wireshark.fullname" . }}-pipeline + - name: wireshark-index-script + configMap: + name: {{ template "wireshark.fullname" . }}-index-script + defaultMode: 0744 + + restartPolicy: Always + dnsPolicy: ClusterFirst + nodeSelector: + nodetype: {{ .Values.nodeSelector.nodetype }} diff --git a/wireshark/helm/templates/wireshark-index-script.yaml b/wireshark/helm/templates/wireshark-index-script.yaml new file mode 100644 index 0000000..57563cb --- /dev/null +++ b/wireshark/helm/templates/wireshark-index-script.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "wireshark.fullname" . }}-index-script +data: + wireshark-index.sh: | + #!/bin/bash + # Creates wireshark index in Elasticsearch + curl -X PUT "data-service:9200/_template/packets" -H 'Content-Type: application/json' -d' + { + "template": "packets-*", + "mappings": { + "pcap_file": { + "dynamic": "false", + "properties": { + "timestamp": { + "type": "date" + }, + "layers": { + "properties": { + "frame": { + "properties": { + "frame_frame_len": { + "type": "long" + }, + "frame_frame_protocols": { + "type": "keyword" + } + } + }, + "ip": { + "properties": { + "ip_ip_src": { + "type": "ip" + }, + "ip_ip_dst": { + "type": "ip" + } + } + }, + "udp": { + "properties": { + "udp_udp_srcport": { + "type": "integer" + }, + "udp_udp_dstport": { + "type": "integer" + } + } + } + } + } + } + } + } + } + ' diff --git a/wireshark/helm/values.yaml b/wireshark/helm/values.yaml new file mode 100644 index 0000000..e57d990 --- /dev/null +++ b/wireshark/helm/values.yaml @@ -0,0 +1,46 @@ +# EDCOP Wireshark Chart values +images: + bro: miked235/wireshark + logstash: docker.elastic.co/logstash/logstash:6.2.4 + redis: redis:4.0.9 + filebeat: docker.elastic.co/beats/filebeat:6.2.4 +networks: + # Overlay is the name of the default cni network + overlay: calico + # Passive is the name of the sriov interface + passive: passive +nodeSelector: + nodetype: worker +wiresharkConfig: + # Insert tshark command line arguments here, i.e. "-j ip" + env: + args: "" + # Requests are set to accommodate limited resource VMs + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 8 + memory: 16G +logstashConfig: + threads: 2 + batchCount: 250 + initialJvmHeap: 4g + maxJvmHeap: 4g + pipelineWorkers: 2 + pipelineOutputWorkers: 2 + pipelineBatchSize: 150 + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 8G +redisConfig: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 2 + memory: 8G +