From 1325139e86103fd379df0ec9cb5558243bd73075 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Wed, 23 Oct 2024 18:11:02 -0400
Subject: [PATCH 01/34] [SECURITY] Eliminate need for root user for log
 collecting pods

---
 CHANGELOG.md                                  |  8 ++++
 logging/bin/deploy_fluentbit_opensearch.sh    |  5 +++
 ...luent-bit_config.configmap_opensearch.yaml | 11 +++--
 .../fb/fluent-bit_helm_values_opensearch.yaml | 42 +++++++++++++++++++
 logging/fb/migrate_fbstate_db.sh              | 24 +++++++++++
 5 files changed, 86 insertions(+), 4 deletions(-)
 create mode 100755 logging/fb/migrate_fbstate_db.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1f2566f..8dfc450e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # SAS Viya Monitoring for Kubernetes
 
+## Unreleased
+* **Logging**
+  * [SECURITY] Fluent Bit log collecting pods no longer run as `root` user and now run with `readOnlyRootFilesystem` 
+set to 'true'. In addition, the database used to maintain state information for the log collector has moved
+to a hostPath volume and renamed. A new initContainer has been added to handle migrating any existing state 
+information and make adjustments to file ownership/permissions. NOTE: This initContainer runs under the `root`
+user but runs only briefly during the initial deployment process.
+
 ## Version 1.2.30 (11OCT2024)
 * **Logging**
   * [SECURITY] OpenSearch Dashboards pod `securityContext` updated to set allowPrivilegeEscalation to 'false'
diff --git a/logging/bin/deploy_fluentbit_opensearch.sh b/logging/bin/deploy_fluentbit_opensearch.sh
index f617b1cd..e36758a2 100755
--- a/logging/bin/deploy_fluentbit_opensearch.sh
+++ b/logging/bin/deploy_fluentbit_opensearch.sh
@@ -159,6 +159,11 @@ kubectl -n $LOG_NS create configmap fb-env-vars \
 
 kubectl -n $LOG_NS label configmap fb-env-vars   managed-by=v4m-es-script
 
+# Create ConfigMap containing Fluent Bit database migration script
+kubectl -n $LOG_NS delete configmap fb-dbmigrate-script --ignore-not-found
+kubectl -n $LOG_NS create configmap fb-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n $LOG_NS label  configmap fb-dbmigrate-script managed-by=v4m-es-script
+
 ## Get Helm Chart Name
 log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
 chart2install="$(get_helmchart_reference $FLUENTBIT_HELM_CHART_REPO $FLUENTBIT_HELM_CHART_NAME $FLUENTBIT_HELM_CHART_VERSION)"
diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index 9c4ed548..d02f64e0 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.26
+       Set  fb_configMap_version 0.2.27noroot2
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -974,6 +974,7 @@ data:
        Allowlist_key  properties
     [INPUT]
        Name             tail
+       Alias            tail4logs
        Path             /var/log/containers/*.log
        Exclude_Path     /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
@@ -984,12 +985,13 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [INPUT]
        # NOTE: ** METALOGGING **
        Name             tail
+       Alias            tail4ops
        Path             /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
        Tag              opsK.*
@@ -998,7 +1000,7 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [OUTPUT]
@@ -1053,7 +1055,8 @@ data:
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
-       storage.path /tmp/fb_buffer/
+       ###storage.path /tmp/fb_buffer/
+       storage.path /var/log/v4m-fb-storage
        storage.checksum off
        storage.sync normal
        storage.backlog.mem_limit 5M
diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index 0d66e6ff..051d5ab5 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -24,7 +24,14 @@ extraVolumeMounts:
 - mountPath: /fluent-bit/etc/viya-tracing.conf
   name: tracing-config
   subPath: viya-tracing.conf
+- mountPath: /var/log/v4m-fb-storage
+  name: v4m-fb-storage
+  readOnly: false
 extraVolumes:
+- hostPath:
+    path: /var/log/v4m-fb-storage
+    type: DirectoryOrCreate
+  name: v4m-fb-storage
 - configMap:
     defaultMode: 420
     name: fb-viya-parsers
@@ -33,6 +40,10 @@ extraVolumes:
     defaultMode: 420
     name: fb-viya-tracing
   name: tracing-config
+- configMap:
+    defaultMode: 0755
+    name: fb-dbmigrate-script
+  name: dbmigrate-script
 fullConfigMap: false
 existingConfigMap: fb-fluent-bit-config
 
@@ -63,3 +74,34 @@ resources:
   requests:
     cpu: 100m
     memory: 128Mi
+
+###podSecurityContext:
+###  fsGroup: 3301     ###23OCT24
+
+securityContext:
+  runAsUser: 3301
+  readOnlyRootFilesystem: true
+  privileged: false
+  capabilities:
+    drop: ["all"]
+    add: ["CHOWN"]
+
+
+initContainers:
+- name: chowner-v4m-fb-storage
+  image: docker.io/library/busybox:latest
+  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
+  securityContext:
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop: ["all"]
+      add: ["CHOWN"]
+    runAsUser: 0
+    runAsNonRoot: false
+  volumeMounts:
+  - name: v4m-fb-storage
+    mountPath: /var/log/v4m-fb-storage
+  - name: dbmigrate-script
+    mountPath: /usr/bin/migrate_fbstate_db.sh
+    readOnly: true
+    subPath: migrate_fbstate_db.sh
diff --git a/logging/fb/migrate_fbstate_db.sh b/logging/fb/migrate_fbstate_db.sh
new file mode 100755
index 00000000..520fe19e
--- /dev/null
+++ b/logging/fb/migrate_fbstate_db.sh
@@ -0,0 +1,24 @@
+#! /bin/sh
+
+# Copyright © 2024, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+newFilename="${NEW_FILENAME:-v4m_fb.db}"
+newDir="${NEW_DIR:-/var/log/v4m-fb-storage}"
+oldFile="${OLD_FILE:-/var/log/sas_viya_flb.db}"
+newFile="$newDir/$newFilename"
+
+if [ ! -f "$newFile" ]; then
+   echo "INFO No existing instance of [$newFile] found"
+   if [ -f "$oldFile" ]; then
+      echo "INFO Migrating [$oldFile] to [$newFile]"
+      cp $oldFile $newFile
+    else
+       echo "INFO No previous instance found"
+   fi
+else
+   echo "INFO An existing instance of [$newFile] found"
+fi
+
+chown -R 3301:3301 $newDir
+

From 91f98251b8951de71c86dcfc9cb3a998c80d63ec Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 1 Nov 2024 14:18:40 -0400
Subject: [PATCH 02/34] Migrate DB, remove configmap

---
 logging/bin/deploy_fluentbit_opensearch.sh        | 10 ++++++++++
 logging/bin/remove_fluentbit_opensearch.sh        |  1 +
 logging/fb/fluent-bit_helm_values_opensearch.yaml |  7 +++++++
 logging/fb/migrate_fbstate_db.sh                  |  9 +++++++--
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/logging/bin/deploy_fluentbit_opensearch.sh b/logging/bin/deploy_fluentbit_opensearch.sh
index e36758a2..71052e54 100755
--- a/logging/bin/deploy_fluentbit_opensearch.sh
+++ b/logging/bin/deploy_fluentbit_opensearch.sh
@@ -159,6 +159,12 @@ kubectl -n $LOG_NS create configmap fb-env-vars \
 
 kubectl -n $LOG_NS label configmap fb-env-vars   managed-by=v4m-es-script
 
+# Check to see if we are upgrading from earlier version requiring root access
+if [ "$( kubectl -n $LOG_NS get configmap fb-dbmigrate-script -o name --ignore-not-found)" != "configmap/fb-dbmigrate-script" ]; then
+   log_debug "An earlier FB configuration requiring 'root detected; stopping FB pod to allow migration"
+   kubectl -n "$LOG_NS" delete daemonset v4m-fb --ignore-not-found
+fi
+
 # Create ConfigMap containing Fluent Bit database migration script
 kubectl -n $LOG_NS delete configmap fb-dbmigrate-script --ignore-not-found
 kubectl -n $LOG_NS create configmap fb-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
@@ -181,8 +187,12 @@ helm $helmDebug upgrade --install --namespace $LOG_NS v4m-fb  \
   --set fullnameOverride=v4m-fb \
   $chart2install
 
+#pause to allow migration script to complete (if necessary)
+sleep 20
+
 #Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
 disable_sa_token_automount $LOG_NS v4m-fb
+# FB pods will restart after following call if automount is not already enabled
 enable_pod_token_automount $LOG_NS daemonset v4m-fb
 
 # Force restart of daemonset to ensure we pick up latest config changes
diff --git a/logging/bin/remove_fluentbit_opensearch.sh b/logging/bin/remove_fluentbit_opensearch.sh
index 3a4dcc9f..acf03fcd 100755
--- a/logging/bin/remove_fluentbit_opensearch.sh
+++ b/logging/bin/remove_fluentbit_opensearch.sh
@@ -27,6 +27,7 @@ kubectl -n $LOG_NS delete configmap fb-fluent-bit-config   --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-viya-parsers        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-viya-tracing        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-env-vars            --ignore-not-found
+kubectl -n $LOG_NS delete configmap fb-dbmigrate-script    --ignore-not-found
 
 log_debug "Script [$this_script] has completed [$(date)]"
 echo ""
diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index 051d5ab5..99f46432 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -28,6 +28,10 @@ extraVolumeMounts:
   name: v4m-fb-storage
   readOnly: false
 extraVolumes:
+- hostPath:
+    path: /var/log
+    #type: DirectoryOrCreate
+  name: var-log
 - hostPath:
     path: /var/log/v4m-fb-storage
     type: DirectoryOrCreate
@@ -105,3 +109,6 @@ initContainers:
     mountPath: /usr/bin/migrate_fbstate_db.sh
     readOnly: true
     subPath: migrate_fbstate_db.sh
+  - mountPath: /var/log
+    name: var-log
+    readOnly: true
diff --git a/logging/fb/migrate_fbstate_db.sh b/logging/fb/migrate_fbstate_db.sh
index 520fe19e..73160b55 100755
--- a/logging/fb/migrate_fbstate_db.sh
+++ b/logging/fb/migrate_fbstate_db.sh
@@ -3,6 +3,7 @@
 # Copyright © 2024, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+cleanup="${CLEANUP:-false}"
 newFilename="${NEW_FILENAME:-v4m_fb.db}"
 newDir="${NEW_DIR:-/var/log/v4m-fb-storage}"
 oldFile="${OLD_FILE:-/var/log/sas_viya_flb.db}"
@@ -10,15 +11,19 @@ newFile="$newDir/$newFilename"
 
 if [ ! -f "$newFile" ]; then
    echo "INFO No existing instance of [$newFile] found"
+
    if [ -f "$oldFile" ]; then
       echo "INFO Migrating [$oldFile] to [$newFile]"
       cp $oldFile $newFile
+      if [ "$cleanup" == "true" ] && [ "$?" == "0" ]; then
+         echo "INFO Deleting [$newFile]"
+         rm -f $newFile
+      fi
     else
-       echo "INFO No previous instance found"
+       echo "INFO No previous instance of [$oldFile] found"
    fi
 else
    echo "INFO An existing instance of [$newFile] found"
 fi
 
 chown -R 3301:3301 $newDir
-

From faac84c5151414d655f8d3c496e009a65a5314fa Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 8 Nov 2024 18:00:35 -0500
Subject: [PATCH 03/34] OpenShift-specific changes (still work-in-progress)

---
 logging/fb/fluent-bit_helm_values_opensearch.yaml | 13 +++++++------
 logging/openshift/values-fluent-bit.yaml          |  3 +++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index 99f46432..ae60e865 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -28,10 +28,10 @@ extraVolumeMounts:
   name: v4m-fb-storage
   readOnly: false
 extraVolumes:
-- hostPath:
-    path: /var/log
-    #type: DirectoryOrCreate
-  name: var-log
+###- hostPath:
+###    path: /var/log
+###    #type: DirectoryOrCreate
+###  name: var-log
 - hostPath:
     path: /var/log/v4m-fb-storage
     type: DirectoryOrCreate
@@ -94,6 +94,7 @@ securityContext:
 initContainers:
 - name: chowner-v4m-fb-storage
   image: docker.io/library/busybox:latest
+  imagePullPolicy: IfNotPresent
   command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
   securityContext:
     readOnlyRootFilesystem: true
@@ -107,8 +108,8 @@ initContainers:
     mountPath: /var/log/v4m-fb-storage
   - name: dbmigrate-script
     mountPath: /usr/bin/migrate_fbstate_db.sh
-    readOnly: true
+    readOnly: false
     subPath: migrate_fbstate_db.sh
   - mountPath: /var/log
-    name: var-log
+    name: varlog
     readOnly: true
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index 21a883af..11d46132 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -2,3 +2,6 @@ securityContext:
   privileged: true
 openShift:
   enabled: true
+  securityContextConstraints:
+    create: false
+    existingName: "privileged"

From 4baf657ae9119c90dabbd7bdb58865ec6759270e Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Wed, 13 Nov 2024 18:57:35 -0500
Subject: [PATCH 04/34] More OpenShift-specific changes

---
 logging/fb/fluent-bit_helm_values_opensearch.yaml | 1 +
 logging/openshift/values-fluent-bit.yaml          | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index ae60e865..81722181 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -97,6 +97,7 @@ initContainers:
   imagePullPolicy: IfNotPresent
   command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
   securityContext:
+    privileged: true
     readOnlyRootFilesystem: true
     capabilities:
       drop: ["all"]
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index 11d46132..cd42f97a 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -3,5 +3,4 @@ securityContext:
 openShift:
   enabled: true
   securityContextConstraints:
-    create: false
-    existingName: "privileged"
+    create: true

From 112d8b0077cbb99b6daa173cdaa05675d4f88725 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 14 Nov 2024 18:26:15 -0500
Subject: [PATCH 05/34] Draft customized SCC

---
 logging/openshift/v4m-fb-v2_scc.yaml | 40 ++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 logging/openshift/v4m-fb-v2_scc.yaml

diff --git a/logging/openshift/v4m-fb-v2_scc.yaml b/logging/openshift/v4m-fb-v2_scc.yaml
new file mode 100644
index 00000000..e43faff7
--- /dev/null
+++ b/logging/openshift/v4m-fb-v2_scc.yaml
@@ -0,0 +1,40 @@
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: v4m-fb-v2
+
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: true
+allowedCapabilities:
+- CHOWN
+defaultAddCapabilities: null
+defaultAllowPrivilegeEscalation: false
+forbiddenSysctls:
+- '*'
+fsGroup:
+  type: RunAsAny
+groups: []
+priority: null
+readOnlyRootFilesystem: false
+requiredDropCapabilities:
+- MKNOD
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: RunAsAny
+users: []
+volumes:
+- configMap
+- downwardAPI
+- emptyDir
+- hostPath
+- persistentVolumeClaim
+- projected
+- secret

From 9792c48efe04e599692141d619ba0721de914b18 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 14 Nov 2024 19:49:58 -0500
Subject: [PATCH 06/34] Rename SCC; add logic to create it as pre-requisite on
 OpenShift

---
 logging/bin/deploy_openshift_prereqs.sh                    | 7 +++++++
 .../{v4m-fb-v2_scc.yaml => fb_v4m-logging-fb_v2_scc.yaml}  | 3 +--
 2 files changed, 8 insertions(+), 2 deletions(-)
 rename logging/openshift/{v4m-fb-v2_scc.yaml => fb_v4m-logging-fb_v2_scc.yaml} (96%)

diff --git a/logging/bin/deploy_openshift_prereqs.sh b/logging/bin/deploy_openshift_prereqs.sh
index 8d57b1c3..90b3bea4 100755
--- a/logging/bin/deploy_openshift_prereqs.sh
+++ b/logging/bin/deploy_openshift_prereqs.sh
@@ -25,6 +25,13 @@ fi
 # link OpenSearch serviceAccounts to 'privileged' scc
 oc adm policy add-scc-to-user privileged -z v4m-os -n $LOG_NS
 
+# create the 'v4m-logging-v2' SCC, if it does not already exist
+if oc get scc v4m-logging-v2 2>/dev/null 1>&2; then
+   log_info "Skipping scc creation; using existing scc [v4m-logging-v2]"
+else
+   oc create -f logging/openshift/fb_v4m-logging-v2_scc.yaml
+fi
+
 log_info "OpenShift Prerequisites have been deployed."
 
 log_debug "Script [$this_script] has completed [$(date)]"
diff --git a/logging/openshift/v4m-fb-v2_scc.yaml b/logging/openshift/fb_v4m-logging-fb_v2_scc.yaml
similarity index 96%
rename from logging/openshift/v4m-fb-v2_scc.yaml
rename to logging/openshift/fb_v4m-logging-fb_v2_scc.yaml
index e43faff7..da2a3a5e 100644
--- a/logging/openshift/v4m-fb-v2_scc.yaml
+++ b/logging/openshift/fb_v4m-logging-fb_v2_scc.yaml
@@ -1,8 +1,7 @@
 apiVersion: security.openshift.io/v1
 kind: SecurityContextConstraints
 metadata:
-  name: v4m-fb-v2
-
+  name: v4m-logging-v2
 allowHostDirVolumePlugin: true
 allowHostIPC: false
 allowHostNetwork: false

From 3f5465ba333dfca287a78ffa5c7597570c304432 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 14 Nov 2024 20:41:13 -0500
Subject: [PATCH 07/34] More OpenShift changes

---
 logging/bin/remove_openshift_artifacts.sh                     | 4 +++-
 ..._v4m-logging-fb_v2_scc.yaml => fb_v4m-logging-v2_scc.yaml} | 0
 logging/openshift/values-fluent-bit-events.yaml               | 2 +-
 logging/openshift/values-fluent-bit.yaml                      | 3 ++-
 4 files changed, 6 insertions(+), 3 deletions(-)
 rename logging/openshift/{fb_v4m-logging-fb_v2_scc.yaml => fb_v4m-logging-v2_scc.yaml} (100%)

diff --git a/logging/bin/remove_openshift_artifacts.sh b/logging/bin/remove_openshift_artifacts.sh
index 7bb35391..79ac51a3 100755
--- a/logging/bin/remove_openshift_artifacts.sh
+++ b/logging/bin/remove_openshift_artifacts.sh
@@ -19,7 +19,9 @@ if [ "$OPENSHIFT_ARTIFACTS_REMOVE" != "true" ]; then
 fi
 
 # remove custom OpenShift SCC
-oc delete scc v4mlogging --ignore-not-found
+oc delete scc v4mlogging     --ignore-not-found
+oc delete scc v4m-logging-v2 --ignore-not-found
+
 
 
 log_info "OpenShift Prerequisites have been removed."
diff --git a/logging/openshift/fb_v4m-logging-fb_v2_scc.yaml b/logging/openshift/fb_v4m-logging-v2_scc.yaml
similarity index 100%
rename from logging/openshift/fb_v4m-logging-fb_v2_scc.yaml
rename to logging/openshift/fb_v4m-logging-v2_scc.yaml
diff --git a/logging/openshift/values-fluent-bit-events.yaml b/logging/openshift/values-fluent-bit-events.yaml
index 21a883af..d3ec975f 100644
--- a/logging/openshift/values-fluent-bit-events.yaml
+++ b/logging/openshift/values-fluent-bit-events.yaml
@@ -1,4 +1,4 @@
 securityContext: 
-  privileged: true
+  privileged: false
 openShift:
   enabled: true
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index cd42f97a..40b02ea6 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -3,4 +3,5 @@ securityContext:
 openShift:
   enabled: true
   securityContextConstraints:
-    create: true
+    create: false
+    existingName: v4m-logging-v2

From 6b5dfe5aeb9770671666def12dab7298e6d8553d Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Tue, 10 Dec 2024 14:06:08 -0500
Subject: [PATCH 08/34] Start of sprint housekeeping

---
 v4m-chart/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/v4m-chart/Chart.yaml b/v4m-chart/Chart.yaml
index 5da034df..a1288c33 100644
--- a/v4m-chart/Chart.yaml
+++ b/v4m-chart/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: v4m
 description: SAS Viya 4 Monitoring for Kubernetes (https://github.com/sassoftware/viya4-monitoring-kubernetes)
 type: application
-version: "1.2.32"
-appVersion: "1.2.32"
+version: "1.2.33-SNAPSHOT"
+appVersion: "1.2.33-SNAPSHOT"

From 88aadf8c6c4ebad7541cb793de0bc01c16b6e1a2 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Tue, 10 Dec 2024 17:15:25 -0500
Subject: [PATCH 09/34] Enable multiline parsing by default

---
 logging/bin/deploy_fluentbit_opensearch.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logging/bin/deploy_fluentbit_opensearch.sh b/logging/bin/deploy_fluentbit_opensearch.sh
index f617b1cd..0c5ec811 100755
--- a/logging/bin/deploy_fluentbit_opensearch.sh
+++ b/logging/bin/deploy_fluentbit_opensearch.sh
@@ -98,7 +98,7 @@ fi
 log_debug "Using FB ConfigMap:" $FB_CONFIGMAP
 
 # Multiline parser setup
-LOG_MULTILINE_ENABLED=${LOG_MULTILINE_ENABLED}
+LOG_MULTILINE_ENABLED=${LOG_MULTILINE_ENABLED:-true}
 if [ "$LOG_MULTILINE_ENABLED" == "true" ]; then
   LOG_MULTILINE_PARSER="docker, cri"
 else

From daabc0c432d94ee94f254fea3c891f82b3d395ff Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Tue, 10 Dec 2024 17:17:58 -0500
Subject: [PATCH 10/34] Improve handling of messages from CrunchyData exporter
 containers

---
 logging/fb/fluent-bit_config.configmap_azmonitor.yaml  |  3 ++-
 logging/fb/fluent-bit_config.configmap_opensearch.yaml |  3 ++-
 logging/fb/viya-parsers.conf                           | 10 +++++++++-
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
index 71ee7dba..0550098f 100644
--- a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
+++ b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.26
+       Set  fb_configMap_version 0.2.27crexp
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -363,6 +363,7 @@ data:
        Match           kube.*
        Key_name        collectapi_message
        Parser          crunchy_poo
+       Parser          crunchy_exporter
        Parser          collectapi
        Parser          collectapiT
        Parser          crunchy_green
diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index 9c4ed548..98c0c1c1 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.26
+       Set  fb_configMap_version 0.2.27crexp
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -363,6 +363,7 @@ data:
        Match           kube.*
        Key_name        collectapi_message
        Parser          crunchy_poo
+       Parser          crunchy_exporter
        Parser          collectapi
        Parser          collectapiT
        Parser          crunchy_green
diff --git a/logging/fb/viya-parsers.conf b/logging/fb/viya-parsers.conf
index 1771028b..bd04b96a 100644
--- a/logging/fb/viya-parsers.conf
+++ b/logging/fb/viya-parsers.conf
@@ -1,4 +1,4 @@
-# Version 1.9.11
+# Version 1.9.12
 [PARSER]
        Name          consul
        Format        regex
@@ -361,3 +361,11 @@
        Regex  (?<timestamp>\d{4}(-\d{2}){2} (\d{2}:?){3}\.\d{3})\s\[(?<level>[A-Z]+)\](?<message>.+)
        Time_Key      timestamp
        Time_format   %F %T.%L
+
+[PARSER]
+       Name          crunchy_exporter
+       # Crunchy v5
+       # Example: ts=2024-12-10T14:14:01.027Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9187
+       Format        regex
+       regex         ts=(?<timestamp>\d{4}(-\d{2}){2}T(\d{2}:?){3})(?<timezone>.*)\s+level=(?<level>\w+)\s+((msg=)"(?<message>.*?)"\s?)?(?<message2>.*)
+

From 526e0978d28ee2780a90a6e95615c04f8e9b3e19 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Tue, 10 Dec 2024 18:16:07 -0500
Subject: [PATCH 11/34] Improve handling of nginx controller messages

---
 logging/fb/fluent-bit_config.configmap_opensearch.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index 98c0c1c1..f7cc23a9 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.27crexp
+       Set  fb_configMap_version 0.2.27crexp2
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -797,6 +797,7 @@ data:
        Parser          calico
        Parser          collectapiT
        Parser          collectapi
+       Parser          k8s-nginx-ingress
        Parser          haproxy_default
        Parser          openvswitch
        Parser          keepalived

From 0e82580109c7368906d680f230c4d38d7a12da4c Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Wed, 11 Dec 2024 11:51:42 -0500
Subject: [PATCH 12/34] Revert "Improve handling of nginx controller messages"

This reverts commit 526e0978d28ee2780a90a6e95615c04f8e9b3e19.
---
 logging/fb/fluent-bit_config.configmap_opensearch.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index f7cc23a9..98c0c1c1 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.27crexp2
+       Set  fb_configMap_version 0.2.27crexp
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -797,7 +797,6 @@ data:
        Parser          calico
        Parser          collectapiT
        Parser          collectapi
-       Parser          k8s-nginx-ingress
        Parser          haproxy_default
        Parser          openvswitch
        Parser          keepalived

From 9baeee7d4610e17e59c9dc5d824988fdc5a6ced3 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Wed, 11 Dec 2024 18:34:54 -0500
Subject: [PATCH 13/34] Add seccompProfile to Fluent Bit SCC

---
 logging/fb/fluent-bit_config.configmap_opensearch.yaml | 2 +-
 logging/openshift/fb_v4mlogging_scc.yaml               | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index 728dea7c..2b61323d 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.27noroot2
+       Set  fb_configMap_version 0.2.27noroot2a
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
diff --git a/logging/openshift/fb_v4mlogging_scc.yaml b/logging/openshift/fb_v4mlogging_scc.yaml
index 65ea612f..d80b74cb 100644
--- a/logging/openshift/fb_v4mlogging_scc.yaml
+++ b/logging/openshift/fb_v4mlogging_scc.yaml
@@ -17,6 +17,8 @@ runAsUser:
   type: RunAsAny
 seLinuxContext:
   type: MustRunAs
+seccompProfiles:
+- runtime/default
 fsGroup:
   type: MustRunAs
 supplementalGroups:

From 6d52037ef75f036becf7d6ac8bc822e663d0939d Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 12 Dec 2024 12:30:12 -0500
Subject: [PATCH 14/34] Move FB initContainer definition to template file to
 support airgap

---
 component_versions.env                        |  1 +
 logging/bin/deploy_fluentbit_opensearch.sh    |  5 +-
 logging/fb/fb_initcontainer_image.template    | 26 +++++++++
 .../fb/fluent-bit_helm_values_opensearch.yaml | 53 ++++++++++---------
 4 files changed, 59 insertions(+), 26 deletions(-)
 create mode 100644 logging/fb/fb_initcontainer_image.template

diff --git a/component_versions.env b/component_versions.env
index 2166f210..6778361a 100644
--- a/component_versions.env
+++ b/component_versions.env
@@ -19,6 +19,7 @@ FLUENTBIT_HELM_CHART_REPO=fluent
 FLUENTBIT_HELM_CHART_NAME=fluent-bit
 FLUENTBIT_HELM_CHART_VERSION=0.47.10
 FB_FULL_IMAGE="cr.fluentbit.io/fluent/fluent-bit:3.1.9"
+FB_INITCONTAINER_FULL_IMAGE="docker.io/library/busybox:latest"
 
 #OpenSearch
 OPENSEARCH_HELM_CHART_REPO=opensearch
diff --git a/logging/bin/deploy_fluentbit_opensearch.sh b/logging/bin/deploy_fluentbit_opensearch.sh
index 73a3f37d..65dbd127 100755
--- a/logging/bin/deploy_fluentbit_opensearch.sh
+++ b/logging/bin/deploy_fluentbit_opensearch.sh
@@ -68,7 +68,8 @@ else
 fi
 
 #Generate yaml file with all container-related keys
-generateImageKeysFile "$FB_FULL_IMAGE"          "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_FULL_IMAGE"                "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_INITCONTAINER_FULL_IMAGE"  "logging/fb/fb_initcontainer_image.template"
 
 # Fluent Bit user customizations
 FB_OPENSEARCH_USER_YAML="${FB_OPENSEARCH_USER_YAML:-$USER_DIR/logging/user-values-fluent-bit-opensearch.yaml}"
@@ -161,7 +162,7 @@ kubectl -n $LOG_NS label configmap fb-env-vars   managed-by=v4m-es-script
 
 # Check to see if we are upgrading from earlier version requiring root access
 if [ "$( kubectl -n $LOG_NS get configmap fb-dbmigrate-script -o name --ignore-not-found)" != "configmap/fb-dbmigrate-script" ]; then
-   log_debug "An earlier FB configuration requiring 'root detected; stopping FB pod to allow migration"
+   log_debug "Removing FB pods (if they exist) to allow migration."
    kubectl -n "$LOG_NS" delete daemonset v4m-fb --ignore-not-found
 fi
 
diff --git a/logging/fb/fb_initcontainer_image.template b/logging/fb/fb_initcontainer_image.template
new file mode 100644
index 00000000..371d1d14
--- /dev/null
+++ b/logging/fb/fb_initcontainer_image.template
@@ -0,0 +1,26 @@
+initContainers:
+- name: chowner-v4m-fb-storage
+  image: __IMAGE_REPO_3LEVEL__:__IMAGE_TAG__
+  ##image: docker.io/library/busybox:latest
+  imagePullPolicy: IfNotPresent
+  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
+  securityContext:
+    privileged: true
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop: ["all"]
+      add: ["CHOWN"]
+    runAsUser: 0
+    runAsNonRoot: false
+  volumeMounts:
+  - name: v4m-fb-storage
+    mountPath: /var/log/v4m-fb-storage
+  - name: dbmigrate-script
+    mountPath: /usr/bin/migrate_fbstate_db.sh
+    readOnly: false
+    subPath: migrate_fbstate_db.sh
+  - mountPath: /var/log
+    name: varlog
+    readOnly: true
+
+
diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index d0573fce..9c253350 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -87,30 +87,35 @@ securityContext:
     drop: ["all"]
     add: ["CHOWN"]
 
-initContainers:
-- name: chowner-v4m-fb-storage
-  image: docker.io/library/busybox:latest
-  imagePullPolicy: IfNotPresent
-  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
-  securityContext:
-    privileged: true
-    readOnlyRootFilesystem: true
-    capabilities:
-      drop: ["all"]
-      add: ["CHOWN"]
-    runAsUser: 0
-    runAsNonRoot: false
-  volumeMounts:
-  - name: v4m-fb-storage
-    mountPath: /var/log/v4m-fb-storage
-  - name: dbmigrate-script
-    mountPath: /usr/bin/migrate_fbstate_db.sh
-    readOnly: false
-    subPath: migrate_fbstate_db.sh
-  - mountPath: /var/log
-    name: varlog
-    readOnly: true
-
 podSecurityContext:
   seccompProfile:
     type: RuntimeDefault
+
+##
+## NOTE: An initContainer is defined in logging/fb/fb_initcontainer_image.template
+##
+
+##initContainers:
+##- name: chowner-v4m-fb-storage
+##  image: docker.io/library/busybox:latest
+##  imagePullPolicy: IfNotPresent
+##  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
+##  securityContext:
+##    privileged: true
+##    readOnlyRootFilesystem: true
+##    capabilities:
+##      drop: ["all"]
+##      add: ["CHOWN"]
+##    runAsUser: 0
+##    runAsNonRoot: false
+##  volumeMounts:
+##  - name: v4m-fb-storage
+##    mountPath: /var/log/v4m-fb-storage
+##  - name: dbmigrate-script
+##    mountPath: /usr/bin/migrate_fbstate_db.sh
+##    readOnly: false
+##    subPath: migrate_fbstate_db.sh
+##  - mountPath: /var/log
+##    name: varlog
+##    readOnly: true
+

From 6da1358345a44560fd5f04fb2ec9d60dfe53c955 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:30:11 -0500
Subject: [PATCH 15/34] Adjust pod/container security settings; tweak OpenShift
 config

---
 .../fb/fluent-bit_helm_values_opensearch.yaml | 33 ++++---------------
 logging/openshift/values-fluent-bit.yaml      |  3 --
 2 files changed, 6 insertions(+), 30 deletions(-)

diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index 9c253350..d780304f 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -83,39 +83,18 @@ securityContext:
   runAsUser: 3301
   readOnlyRootFilesystem: true
   privileged: false
+  allowPrivilegeEscalation: false
   capabilities:
     drop: ["all"]
-    add: ["CHOWN"]
 
 podSecurityContext:
+  runAsNonRoot: true
   seccompProfile:
     type: RuntimeDefault
 
 ##
-## NOTE: An initContainer is defined in logging/fb/fb_initcontainer_image.template
+## NOTE: An initContainer is also deployed; it is defined
+##       in logging/fb/fb_initcontainer_image.template
+##       which is passed to the Helm install command
+##       after value substitution.
 ##
-
-##initContainers:
-##- name: chowner-v4m-fb-storage
-##  image: docker.io/library/busybox:latest
-##  imagePullPolicy: IfNotPresent
-##  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
-##  securityContext:
-##    privileged: true
-##    readOnlyRootFilesystem: true
-##    capabilities:
-##      drop: ["all"]
-##      add: ["CHOWN"]
-##    runAsUser: 0
-##    runAsNonRoot: false
-##  volumeMounts:
-##  - name: v4m-fb-storage
-##    mountPath: /var/log/v4m-fb-storage
-##  - name: dbmigrate-script
-##    mountPath: /usr/bin/migrate_fbstate_db.sh
-##    readOnly: false
-##    subPath: migrate_fbstate_db.sh
-##  - mountPath: /var/log
-##    name: varlog
-##    readOnly: true
-
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index 85ad3626..718a5880 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -1,6 +1,3 @@
-podSecurityContext: null
-securityContext: 
-  privileged: true
 openShift:
   enabled: true
   securityContextConstraints:

From 74ba92514b3b244a8ba7990e5f98cc7e61730828 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 12 Dec 2024 18:32:18 -0500
Subject: [PATCH 16/34] Further tweaks

---
 logging/fb/fb_initcontainer_image.template      | 1 +
 logging/openshift/values-fluent-bit-events.yaml | 1 +
 logging/openshift/values-fluent-bit.yaml        | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/logging/fb/fb_initcontainer_image.template b/logging/fb/fb_initcontainer_image.template
index 371d1d14..4d4b69c0 100644
--- a/logging/fb/fb_initcontainer_image.template
+++ b/logging/fb/fb_initcontainer_image.template
@@ -6,6 +6,7 @@ initContainers:
   command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
   securityContext:
     privileged: true
+    allowPrivilegeEscalation: true
     readOnlyRootFilesystem: true
     capabilities:
       drop: ["all"]
diff --git a/logging/openshift/values-fluent-bit-events.yaml b/logging/openshift/values-fluent-bit-events.yaml
index e65747ea..8984f95b 100644
--- a/logging/openshift/values-fluent-bit-events.yaml
+++ b/logging/openshift/values-fluent-bit-events.yaml
@@ -1,4 +1,5 @@
 podSecurityContext: null
+
 securityContext: 
   privileged: false
 openShift:
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index 718a5880..f3d1fb9d 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -1,3 +1,5 @@
+podSecurityContext: null
+
 openShift:
   enabled: true
   securityContextConstraints:

From f92e3cf20cf9a066bbf07006a210e3988a1fd31d Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:36:16 -0500
Subject: [PATCH 17/34] OpenShift Good - FB Log Collector

---
 logging/fb/fluent-bit_helm_values_opensearch.yaml | 2 +-
 logging/openshift/fb_v4m-logging-v2_scc.yaml      | 2 ++
 logging/openshift/values-fluent-bit.yaml          | 4 +++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index d780304f..63d45246 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -85,7 +85,7 @@ securityContext:
   privileged: false
   allowPrivilegeEscalation: false
   capabilities:
-    drop: ["all"]
+    drop: ["ALL"]
 
 podSecurityContext:
   runAsNonRoot: true
diff --git a/logging/openshift/fb_v4m-logging-v2_scc.yaml b/logging/openshift/fb_v4m-logging-v2_scc.yaml
index da2a3a5e..8bb8f87a 100644
--- a/logging/openshift/fb_v4m-logging-v2_scc.yaml
+++ b/logging/openshift/fb_v4m-logging-v2_scc.yaml
@@ -26,6 +26,8 @@ runAsUser:
   type: RunAsAny
 seLinuxContext:
   type: MustRunAs
+seccompProfiles:
+- runtime/default
 supplementalGroups:
   type: RunAsAny
 users: []
diff --git a/logging/openshift/values-fluent-bit.yaml b/logging/openshift/values-fluent-bit.yaml
index f3d1fb9d..76ed8276 100644
--- a/logging/openshift/values-fluent-bit.yaml
+++ b/logging/openshift/values-fluent-bit.yaml
@@ -1,4 +1,6 @@
-podSecurityContext: null
+securityContext:
+  privileged: true
+  allowPrivilegeEscalation: true
 
 openShift:
   enabled: true

From 81f172138dad91f105627628317a5d22c6faa8c1 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:44:49 -0500
Subject: [PATCH 18/34] Remove changes to obsolete file

---
 logging/openshift/fb_v4mlogging_scc.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/logging/openshift/fb_v4mlogging_scc.yaml b/logging/openshift/fb_v4mlogging_scc.yaml
index d80b74cb..65ea612f 100644
--- a/logging/openshift/fb_v4mlogging_scc.yaml
+++ b/logging/openshift/fb_v4mlogging_scc.yaml
@@ -17,8 +17,6 @@ runAsUser:
   type: RunAsAny
 seLinuxContext:
   type: MustRunAs
-seccompProfiles:
-- runtime/default
 fsGroup:
   type: MustRunAs
 supplementalGroups:

From cab2bfe3739a75f23e4c5105a0c0191d5c0cabcd Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 12:20:43 -0500
Subject: [PATCH 19/34] FB as K8s Event collector - security tweaks

---
 logging/bin/deploy_openshift_prereqs.sh       |  7 ++++
 logging/fb/fluent-bit_helm_values_events.yaml |  2 +
 logging/openshift/fb_v4m-k8sevents_scc.yaml   | 40 +++++++++++++++++++
 .../openshift/values-fluent-bit-events.yaml   |  8 ++--
 4 files changed, 53 insertions(+), 4 deletions(-)
 create mode 100644 logging/openshift/fb_v4m-k8sevents_scc.yaml

diff --git a/logging/bin/deploy_openshift_prereqs.sh b/logging/bin/deploy_openshift_prereqs.sh
index 90b3bea4..35cbf285 100755
--- a/logging/bin/deploy_openshift_prereqs.sh
+++ b/logging/bin/deploy_openshift_prereqs.sh
@@ -32,6 +32,13 @@ else
    oc create -f logging/openshift/fb_v4m-logging-v2_scc.yaml
 fi
 
+# create the 'v4m-k8sevents' SCC, if it does not already exist
+if oc get scc v4m-k8sevents 2>/dev/null 1>&2; then
+   log_info "Skipping scc creation; using existing scc [v4m-k8sevents]"
+else
+   oc create -f logging/openshift/fb_v4m-k8sevents_scc.yaml
+fi
+
 log_info "OpenShift Prerequisites have been deployed."
 
 log_debug "Script [$this_script] has completed [$(date)]"
diff --git a/logging/fb/fluent-bit_helm_values_events.yaml b/logging/fb/fluent-bit_helm_values_events.yaml
index d1b56f19..5a009db2 100644
--- a/logging/fb/fluent-bit_helm_values_events.yaml
+++ b/logging/fb/fluent-bit_helm_values_events.yaml
@@ -34,6 +34,8 @@ extraVolumeMounts:
   subPath: v4m_fb_events.db
 
 securityContext:
+  privileged: false
+  allowPrivilegeEscalation: false
   runAsNonRoot: true
   runAsUser: 1001
   readOnlyRootFilesystem: true
diff --git a/logging/openshift/fb_v4m-k8sevents_scc.yaml b/logging/openshift/fb_v4m-k8sevents_scc.yaml
new file mode 100644
index 00000000..b6c12c6f
--- /dev/null
+++ b/logging/openshift/fb_v4m-k8sevents_scc.yaml
@@ -0,0 +1,40 @@
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: v4m-k8sevents
+allowHostDirVolumePlugin: true
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: true
+allowedCapabilities: []
+defaultAddCapabilities: null
+defaultAllowPrivilegeEscalation: false
+forbiddenSysctls:
+- '*'
+fsGroup:
+  type: RunAsAny
+groups: []
+priority: null
+readOnlyRootFilesystem: false
+requiredDropCapabilities:
+- MKNOD
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: MustRunAs
+seccompProfiles:
+- runtime/default
+supplementalGroups:
+  type: RunAsAny
+users: []
+volumes:
+- configMap
+- downwardAPI
+- emptyDir
+- hostPath
+- persistentVolumeClaim
+- projected
+- secret
diff --git a/logging/openshift/values-fluent-bit-events.yaml b/logging/openshift/values-fluent-bit-events.yaml
index 8984f95b..05e60226 100644
--- a/logging/openshift/values-fluent-bit-events.yaml
+++ b/logging/openshift/values-fluent-bit-events.yaml
@@ -1,6 +1,6 @@
-podSecurityContext: null
-
-securityContext: 
-  privileged: false
 openShift:
   enabled: true
+  securityContextConstraints:
+    create: false
+    existingName: v4m-k8sevents
+

From 03d3efe2e25819de30934d4af0d4ed5d9f9baf0a Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 12:22:36 -0500
Subject: [PATCH 20/34] Remove v4m-k8sevents SCC during cleanup

---
 logging/bin/remove_openshift_artifacts.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/logging/bin/remove_openshift_artifacts.sh b/logging/bin/remove_openshift_artifacts.sh
index 79ac51a3..cd71fe47 100755
--- a/logging/bin/remove_openshift_artifacts.sh
+++ b/logging/bin/remove_openshift_artifacts.sh
@@ -21,6 +21,7 @@ fi
 # remove custom OpenShift SCC
 oc delete scc v4mlogging     --ignore-not-found
 oc delete scc v4m-logging-v2 --ignore-not-found
+oc delete scc v4m-k8sevents  --ignore-not-found
 
 
 

From bc04657bf1cdbb8edfb3484afed3cc79f0f5d0d5 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 12:43:45 -0500
Subject: [PATCH 21/34] Update CHANGELOG.md

---
 CHANGELOG.md | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1d071426..53f5b319 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,11 +2,15 @@
 
 ## Unreleased
 * **Logging**
-  * [SECURITY] Fluent Bit log collecting pods no longer run as `root` user and now run with `readOnlyRootFilesystem` 
-set to 'true'. In addition, the database used to maintain state information for the log collector has moved
-to a hostPath volume and renamed. A new initContainer has been added to handle migrating any existing state 
-information and make adjustments to file ownership/permissions. NOTE: This initContainer runs under the `root`
-user but runs only briefly during the initial deployment process.
+  * [SECURITY] Fluent Bit log collecting pods no longer run as `root` user.  In addition, the database used to
+maintain state information for the log collector has moved to a hostPath volume and renamed. A new initContainer
+has been added to handle migrating any existing state information and make adjustments to file ownership/permissions.
+NOTE: This initContainer runs under as `root` user but only runs briefly during the initial deployment process.
+  * [SECURITY] Runtime security controls for all Fluent Bit pods (inc. both log collecting and Kubernetes event
+collecting) have been tightened.  Changes include: adding seecompProfile; and disallowing privileged containers,
+privilege escalation and write access to root filesystem.
+  * [SECURITY] On OpenShift, all Fluent Bit pods now use custom SCC objects to support changes described above.
+  * [CHANGE] Improved handling of long log messages and those from some Crunchy Data pods
 
 
 ## Version 1.2.32 (09DEC2024)

From 636337b17ec58ba7e90a14512299f776037858bb Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 14:11:19 -0500
Subject: [PATCH 22/34] Update FB for alternate solution (Azure Monitor)

---
 logging/bin/deploy_fluentbit_azmonitor.sh     | 20 +++++++++++++++++--
 logging/bin/remove_fluentbit_azmonitor.sh     |  2 +-
 ...fluent-bit_config.configmap_azmonitor.yaml |  8 +++++---
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/logging/bin/deploy_fluentbit_azmonitor.sh b/logging/bin/deploy_fluentbit_azmonitor.sh
index 225584d6..f5ae35ab 100755
--- a/logging/bin/deploy_fluentbit_azmonitor.sh
+++ b/logging/bin/deploy_fluentbit_azmonitor.sh
@@ -39,7 +39,8 @@ fi
 log_info "Deploying Fluent Bit (Azure Monitor)"
 
 #Generate yaml file with all container-related keys#Generate yaml file with all container-related keys
-generateImageKeysFile "$FB_FULL_IMAGE"          "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_FULL_IMAGE"               "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_INITCONTAINER_FULL_IMAGE" "logging/fb/fb_initcontainer_image.template"
 
 # Fluent Bit user customizations
 FB_AZMONITOR_USER_YAML="${FB_AZMONITOR_USER_YAML:-$USER_DIR/logging/user-values-fluent-bit-azmonitor.yaml}"
@@ -94,7 +95,7 @@ else
 fi
 
 # Multiline parser setup
-LOG_MULTILINE_ENABLED="${LOG_MULTILINE_ENABLED}"
+LOG_MULTILINE_ENABLED="${LOG_MULTILINE_ENABLED:-true}"
 if [ "$LOG_MULTILINE_ENABLED" == "true" ]; then
   LOG_MULTILINE_PARSER="docker, cri"
 else
@@ -154,6 +155,18 @@ kubectl -n $LOG_NS create configmap fbaz-env-vars \
 
 kubectl -n $LOG_NS label configmap fbaz-env-vars   managed-by=v4m-es-script
 
+# Check to see if we are upgrading from earlier version requiring root access
+if [ "$( kubectl -n $LOG_NS get configmap fbaz-dbmigrate-script -o name --ignore-not-found)" != "configmap/fbaz-dbmigrate-script" ]; then
+   log_debug "Removing FB pods (if they exist) to allow migration."
+   kubectl -n "$LOG_NS" delete daemonset v4m-fbaz --ignore-not-found
+fi
+
+# Create ConfigMap containing Fluent Bit database migration script
+kubectl -n $LOG_NS delete configmap fbaz-dbmigrate-script --ignore-not-found
+kubectl -n $LOG_NS create configmap fbaz-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n $LOG_NS label  configmap fbaz-dbmigrate-script managed-by=v4m-es-script
+
+
  ## Get Helm Chart Name
  log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
  chart2install="$(get_helmchart_reference $FLUENTBIT_HELM_CHART_REPO $FLUENTBIT_HELM_CHART_NAME $FLUENTBIT_HELM_CHART_VERSION)"
@@ -170,9 +183,12 @@ helm $helmDebug upgrade --install v4m-fbaz  --namespace $LOG_NS  \
   --set fullnameOverride=v4m-fbaz \
   $chart2install
 
+#pause to allow migration script to complete (if necessary)
+sleep 20
 
 #Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
 disable_sa_token_automount $LOG_NS v4m-fbaz
+# FB pods will restart after following call if automount is not already enabled
 enable_pod_token_automount $LOG_NS daemonset v4m-fbaz
 
 # Force restart of daemonset to ensure we pick up latest config changes
diff --git a/logging/bin/remove_fluentbit_azmonitor.sh b/logging/bin/remove_fluentbit_azmonitor.sh
index c1704e25..e0b5d3d7 100755
--- a/logging/bin/remove_fluentbit_azmonitor.sh
+++ b/logging/bin/remove_fluentbit_azmonitor.sh
@@ -30,7 +30,7 @@ kubectl -n $LOG_NS delete configmap fbaz-fluent-bit-config   --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-viya-parsers        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-viya-tracing        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-env-vars            --ignore-not-found
-
+kubectl -n $LOG_NS delete configmap fbaz-dbmigrate-script    --ignore-not-found
 
 # Should we leave secret in place?
 log_info "Removing Connection information  (secret)"
diff --git a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
index 0550098f..12518998 100644
--- a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
+++ b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.27crexp
+       Set  fb_configMap_version 0.2.27noroot2a
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -973,6 +973,7 @@ data:
        Allowlist_key  properties
     [INPUT]
        Name             tail
+       Alias            tail4logs
        Path             /var/log/containers/*.log
        Exclude_Path     /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
@@ -983,12 +984,13 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [INPUT]
        # NOTE: ** METALOGGING **
        Name             tail
+       Alias            tail4ops
        Path             /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
        Tag              opsK.*
@@ -997,7 +999,7 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [OUTPUT]

From 18c1c4c07cc42c03870b884b582a863034a6bacc Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 14:21:43 -0500
Subject: [PATCH 23/34] Futher tweaks re: alternate solution

---
 .../fb/fluent-bit_helm_values_azmonitor.yaml  | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/logging/fb/fluent-bit_helm_values_azmonitor.yaml b/logging/fb/fluent-bit_helm_values_azmonitor.yaml
index 9f298d37..28f73744 100644
--- a/logging/fb/fluent-bit_helm_values_azmonitor.yaml
+++ b/logging/fb/fluent-bit_helm_values_azmonitor.yaml
@@ -31,7 +31,14 @@ extraVolumeMounts:
 - mountPath: /fluent-bit/etc/viya-tracing.conf
   name: tracing-config
   subPath: viya-tracing.conf
+- mountPath: /var/log/v4m-fb-storage
+  name: v4m-fb-storage
+  readOnly: false
 extraVolumes:
+- hostPath:
+    path: /var/log/v4m-fb-storage
+    type: DirectoryOrCreate
+  name: v4m-fb-storage
 - configMap:
     defaultMode: 420
     name: fbaz-viya-parsers
@@ -40,6 +47,10 @@ extraVolumes:
     defaultMode: 420
     name: fbaz-viya-tracing
   name: tracing-config
+- configMap:
+    defaultMode: 0755
+    name: fbaz-dbmigrate-script
+  name: dbmigrate-script
 fullConfigMap: false
 existingConfigMap: fbaz-fluent-bit-config
 
@@ -70,6 +81,22 @@ resources:
     cpu: 100m
     memory: 128Mi
 
+securityContext:
+  runAsUser: 3301
+  readOnlyRootFilesystem: true
+  privileged: false
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop: ["ALL"]
+
 podSecurityContext:
+  runAsNonRoot: true
   seccompProfile:
     type: RuntimeDefault
+
+##
+## NOTE: An initContainer is also deployed; it is defined
+##       in logging/fb/fb_initcontainer_image.template
+##       which is passed to the Helm install command
+##       after value substitution.
+##

From 0f4e1d132992229a4943fe9004b465d1a2d78bd4 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 15:52:20 -0500
Subject: [PATCH 24/34] Further tweaks (2): alternate solution

---
 logging/fb/fluent-bit_config.configmap_azmonitor.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
index 12518998..aa37c5a3 100644
--- a/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
+++ b/logging/fb/fluent-bit_config.configmap_azmonitor.yaml
@@ -1018,7 +1018,7 @@ data:
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
-       storage.path /tmp/fb_buffer/
+       storage.path /var/log/v4m-fb-storage
        storage.checksum off
        storage.sync normal
        storage.backlog.mem_limit 5M

From 1fc0c7f8721d5e5bbc2c355aab0f55dc741ce89a Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 13 Dec 2024 15:53:15 -0500
Subject: [PATCH 25/34] Revise ARTIFACT_INVENTORY.template to mention FB
 initContainer

---
 .github/workflows/build-artifact/ARTIFACT_INVENTORY.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-artifact/ARTIFACT_INVENTORY.template b/.github/workflows/build-artifact/ARTIFACT_INVENTORY.template
index afaa7e3e..7323c238 100644
--- a/.github/workflows/build-artifact/ARTIFACT_INVENTORY.template
+++ b/.github/workflows/build-artifact/ARTIFACT_INVENTORY.template
@@ -16,9 +16,9 @@ registry/repository/image_name:version
 
 | Subsystem| Component | Fully Qualified Container-Image Name (registry/repository/image_name:version)|
 |----|----|----|
-| Logging | BusyBox (OpenSearch) | __OS_SYSCTL_FULL_IMAGE__ |
 | Logging | Fluent Bit | __FB_FULL_IMAGE__ |
 | Logging | Elasticsearch Exporter | __ES_EXPORTER_FULL_IMAGE__ |
+| Logging | initContainer (Fluent Bit, OpenSearch) | __OS_SYSCTL_FULL_IMAGE__ |
 | Logging | OpenSearch | __OS_FULL_IMAGE__ |
 | Logging | OpenSearch Dashboards| __OSD_FULL_IMAGE__ |
 | Metrics | Alertmanager | __ALERTMANAGER_FULL_IMAGE__ |

From 4f0e718cb289d5035acb00136f7b11dfc11f8b2f Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Tue, 17 Dec 2024 14:34:40 -0500
Subject: [PATCH 26/34] Remove cleanup logic from migrate script

---
 logging/fb/migrate_fbstate_db.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/logging/fb/migrate_fbstate_db.sh b/logging/fb/migrate_fbstate_db.sh
index 73160b55..57b47602 100755
--- a/logging/fb/migrate_fbstate_db.sh
+++ b/logging/fb/migrate_fbstate_db.sh
@@ -1,9 +1,8 @@
 #! /bin/sh
 
-# Copyright © 2024, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
+# Copyright © 2024, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-cleanup="${CLEANUP:-false}"
 newFilename="${NEW_FILENAME:-v4m_fb.db}"
 newDir="${NEW_DIR:-/var/log/v4m-fb-storage}"
 oldFile="${OLD_FILE:-/var/log/sas_viya_flb.db}"
@@ -15,10 +14,6 @@ if [ ! -f "$newFile" ]; then
    if [ -f "$oldFile" ]; then
       echo "INFO Migrating [$oldFile] to [$newFile]"
       cp $oldFile $newFile
-      if [ "$cleanup" == "true" ] && [ "$?" == "0" ]; then
-         echo "INFO Deleting [$newFile]"
-         rm -f $newFile
-      fi
     else
        echo "INFO No previous instance of [$oldFile] found"
    fi

From b4191acf24bf110319e0b6bc69c507f6051ea232 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:55:53 -0500
Subject: [PATCH 27/34] Cleanup

---
 CHANGELOG.md                                           | 4 ++--
 logging/fb/fluent-bit_config.configmap_opensearch.yaml | 1 -
 logging/fb/fluent-bit_helm_values_opensearch.yaml      | 4 ----
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 53f5b319..9c71b5ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,12 +3,12 @@
 ## Unreleased
 * **Logging**
   * [SECURITY] Fluent Bit log collecting pods no longer run as `root` user.  In addition, the database used to
-maintain state information for the log collector has moved to a hostPath volume and renamed. A new initContainer
+maintain state information for the log collector has moved to a hostPath volume and been renamed. A new initContainer
 has been added to handle migrating any existing state information and make adjustments to file ownership/permissions.
 NOTE: This initContainer runs under as `root` user but only runs briefly during the initial deployment process.
   * [SECURITY] Runtime security controls for all Fluent Bit pods (inc. both log collecting and Kubernetes event
 collecting) have been tightened.  Changes include: adding seecompProfile; and disallowing privileged containers,
-privilege escalation and write access to root filesystem.
+privilege escalation and write access to the root filesystem.
   * [SECURITY] On OpenShift, all Fluent Bit pods now use custom SCC objects to support changes described above.
   * [CHANGE] Improved handling of long log messages and those from some Crunchy Data pods
 
diff --git a/logging/fb/fluent-bit_config.configmap_opensearch.yaml b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
index 2b61323d..83ea3e5a 100644
--- a/logging/fb/fluent-bit_config.configmap_opensearch.yaml
+++ b/logging/fb/fluent-bit_config.configmap_opensearch.yaml
@@ -1056,7 +1056,6 @@ data:
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
-       ###storage.path /tmp/fb_buffer/
        storage.path /var/log/v4m-fb-storage
        storage.checksum off
        storage.sync normal
diff --git a/logging/fb/fluent-bit_helm_values_opensearch.yaml b/logging/fb/fluent-bit_helm_values_opensearch.yaml
index 63d45246..d334c4bb 100644
--- a/logging/fb/fluent-bit_helm_values_opensearch.yaml
+++ b/logging/fb/fluent-bit_helm_values_opensearch.yaml
@@ -28,10 +28,6 @@ extraVolumeMounts:
   name: v4m-fb-storage
   readOnly: false
 extraVolumes:
-###- hostPath:
-###    path: /var/log
-###    #type: DirectoryOrCreate
-###  name: var-log
 - hostPath:
     path: /var/log/v4m-fb-storage
     type: DirectoryOrCreate

From e59d7d5e2736dbedd15be5c7e1e5e76e25b2cad3 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 19 Dec 2024 15:11:50 -0500
Subject: [PATCH 28/34] Cleanup based on code review comments

---
 CHANGELOG.md                               |  2 +-
 logging/bin/deploy_fluentbit_azmonitor.sh  | 26 +++++++++++-----------
 logging/bin/deploy_fluentbit_opensearch.sh | 21 ++++++++---------
 3 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9c71b5ae..f4c64e2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,7 @@ maintain state information for the log collector has moved to a hostPath volume
 has been added to handle migrating any existing state information and make adjustments to file ownership/permissions.
 NOTE: This initContainer runs under as `root` user but only runs briefly during the initial deployment process.
   * [SECURITY] Runtime security controls for all Fluent Bit pods (inc. both log collecting and Kubernetes event
-collecting) have been tightened.  Changes include: adding seecompProfile; and disallowing privileged containers,
+collecting) have been tightened.  Changes include: adding seccompProfile; and disallowing privileged containers,
 privilege escalation and write access to the root filesystem.
   * [SECURITY] On OpenShift, all Fluent Bit pods now use custom SCC objects to support changes described above.
   * [CHANGE] Improved handling of long log messages and those from some Crunchy Data pods
diff --git a/logging/bin/deploy_fluentbit_azmonitor.sh b/logging/bin/deploy_fluentbit_azmonitor.sh
index f5ae35ab..33ec9ab7 100755
--- a/logging/bin/deploy_fluentbit_azmonitor.sh
+++ b/logging/bin/deploy_fluentbit_azmonitor.sh
@@ -67,7 +67,7 @@ if [ "$(kubectl -n $LOG_NS get secret connection-info-azmonitor -o name 2>/dev/n
 
    if [ "$AZMONITOR_CUSTOMER_ID" != "NotProvided"  ] && [ "$AZMONITOR_SHARED_KEY" != "NotProvided" ]; then
       log_info "Creating secret [connection-info-azmonitor] in [$LOG_NS] namespace to hold Azure connection information."
-      kubectl -n $LOG_NS create secret generic connection-info-azmonitor --from-literal=customer_id=$AZMONITOR_CUSTOMER_ID --from-literal=shared_key=$AZMONITOR_SHARED_KEY
+      kubectl -n "$LOG_NS" create secret generic connection-info-azmonitor --from-literal=customer_id="$AZMONITOR_CUSTOMER_ID" --from-literal=shared_key="$AZMONITOR_SHARED_KEY"
    else
       log_error "Unable to create secret [$LOG_NS/connection-info-azmonitor] because missing required information: [AZMONITOR_CUSTOMER_ID: $AZMONITOR_CUSTOMER_ID ; AZMONITOR_SHARED_KEY: $AZMONITOR_SHARED_KEY]."
       log_error "You must provide this information via environment variables or create the secret [connection-info-azmonitor] before running this script."
@@ -75,14 +75,14 @@ if [ "$(kubectl -n $LOG_NS get secret connection-info-azmonitor -o name 2>/dev/n
    fi
 else
    log_info "Obtaining connection information from existing secret [$LOG_NS/connection-info-azmonitor]"
-   export AZMONITOR_CUSTOMER_ID=$(kubectl -n $LOG_NS get secret connection-info-azmonitor -o=jsonpath="{.data.customer_id}" |base64 --decode)
-   export AZMONITOR_SHARED_KEY=$(kubectl -n $LOG_NS get secret connection-info-azmonitor -o=jsonpath="{.data.shared_key}" |base64 --decode)
+   export AZMONITOR_CUSTOMER_ID=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.customer_id}" |base64 --decode)
+   export AZMONITOR_SHARED_KEY=$(kubectl -n "$LOG_NS" get secret connection-info-azmonitor -o=jsonpath="{.data.shared_key}" |base64 --decode)
 fi
 
 # Check for an existing Helm release of stable/fluent-bit
 if helm3ReleaseExists fbaz $LOG_NS; then
    log_info "Removing an existing release of deprecated stable/fluent-bit Helm chart from from the [$LOG_NS] namespace [$(date)]"
-   helm  $helmDebug  delete -n $LOG_NS fbaz
+   helm  $helmDebug  delete -n "$LOG_NS" fbaz
 
    if [ $(kubectl get servicemonitors -A |grep fluent-bit-v2 -c) -ge 1 ]; then
       log_debug "Updated serviceMonitor [fluent-bit-v2] appears to be deployed."
@@ -103,11 +103,11 @@ else
 fi
 
 # Create ConfigMap containing Fluent Bit configuration
-kubectl -n $LOG_NS apply -f $FB_CONFIGMAP
+kubectl -n "$LOG_NS" apply -f $FB_CONFIGMAP
 
 # Create ConfigMap containing Viya-customized parsers (delete it first)
-kubectl -n $LOG_NS delete configmap fbaz-viya-parsers --ignore-not-found
-kubectl -n $LOG_NS create configmap fbaz-viya-parsers  --from-file=logging/fb/viya-parsers.conf
+kubectl -n "$LOG_NS" delete configmap fbaz-viya-parsers --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fbaz-viya-parsers  --from-file=logging/fb/viya-parsers.conf
 
 TRACING_ENABLE="${TRACING_ENABLE:-false}"
 if [ "$TRACING_ENABLE" == "true" ]; then
@@ -147,13 +147,13 @@ fi
 MON_NS="${MON_NS:-monitoring}"
 
 # Create ConfigMap containing Kubernetes container runtime log format
-kubectl -n $LOG_NS delete configmap fbaz-env-vars --ignore-not-found
-kubectl -n $LOG_NS create configmap fbaz-env-vars \
+kubectl -n "$LOG_NS" delete configmap fbaz-env-vars --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fbaz-env-vars \
                    --from-literal=KUBERNETES_RUNTIME_LOGFMT=$KUBERNETES_RUNTIME_LOGFMT \
                    --from-literal=LOG_MULTILINE_PARSER="${LOG_MULTILINE_PARSER}" \
                    --from-literal=MON_NS="${MON_NS}"
 
-kubectl -n $LOG_NS label configmap fbaz-env-vars   managed-by=v4m-es-script
+kubectl -n "$LOG_NS" label configmap fbaz-env-vars   managed-by=v4m-es-script
 
 # Check to see if we are upgrading from earlier version requiring root access
 if [ "$( kubectl -n $LOG_NS get configmap fbaz-dbmigrate-script -o name --ignore-not-found)" != "configmap/fbaz-dbmigrate-script" ]; then
@@ -162,9 +162,9 @@ if [ "$( kubectl -n $LOG_NS get configmap fbaz-dbmigrate-script -o name --ignore
 fi
 
 # Create ConfigMap containing Fluent Bit database migration script
-kubectl -n $LOG_NS delete configmap fbaz-dbmigrate-script --ignore-not-found
-kubectl -n $LOG_NS create configmap fbaz-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
-kubectl -n $LOG_NS label  configmap fbaz-dbmigrate-script managed-by=v4m-es-script
+kubectl -n "$LOG_NS" delete configmap fbaz-dbmigrate-script --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fbaz-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n "$LOG_NS" label  configmap fbaz-dbmigrate-script managed-by=v4m-es-script
 
 
  ## Get Helm Chart Name
diff --git a/logging/bin/deploy_fluentbit_opensearch.sh b/logging/bin/deploy_fluentbit_opensearch.sh
index 65dbd127..b511a332 100755
--- a/logging/bin/deploy_fluentbit_opensearch.sh
+++ b/logging/bin/deploy_fluentbit_opensearch.sh
@@ -55,7 +55,7 @@ helm2ReleaseCheck fb-$LOG_NS
 # Check for an existing Helm release of stable/fluent-bit
 if helm3ReleaseExists fb $LOG_NS; then
    log_verbose "Removing an existing release of deprecated stable/fluent-bit Helm chart from from the [$LOG_NS] namespace [$(date)]"
-   helm  $helmDebug  delete -n $LOG_NS fb
+   helm  $helmDebug  delete -n "$LOG_NS" fb
 
    if [ $(kubectl get servicemonitors -A |grep fluent-bit-v2 -c) -ge 1 ]; then
       log_debug "Updated serviceMonitor [fluent-bit-v2] appears to be deployed."
@@ -107,11 +107,11 @@ else
 fi
 
 # Create ConfigMap containing Fluent Bit configuration
-kubectl -n $LOG_NS apply -f $FB_CONFIGMAP
+kubectl -n "$LOG_NS" apply -f $FB_CONFIGMAP
 
 # Create ConfigMap containing Viya-customized parsers (delete it first)
-kubectl -n $LOG_NS delete configmap fb-viya-parsers --ignore-not-found
-kubectl -n $LOG_NS create configmap fb-viya-parsers  --from-file=logging/fb/viya-parsers.conf
+kubectl -n "$LOG_NS" delete configmap fb-viya-parsers --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fb-viya-parsers  --from-file=logging/fb/viya-parsers.conf
 
 TRACING_ENABLE="${TRACING_ENABLE:-false}"
 if [ "$TRACING_ENABLE" == "true" ]; then
@@ -151,14 +151,14 @@ fi
 MON_NS="${MON_NS:-monitoring}"
 
 # Create ConfigMap containing Kubernetes container runtime log format
-kubectl -n $LOG_NS delete configmap fb-env-vars --ignore-not-found
-kubectl -n $LOG_NS create configmap fb-env-vars \
+kubectl -n "$LOG_NS" delete configmap fb-env-vars --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fb-env-vars \
                    --from-literal=KUBERNETES_RUNTIME_LOGFMT="$KUBERNETES_RUNTIME_LOGFMT" \
                    --from-literal=LOG_MULTILINE_PARSER="${LOG_MULTILINE_PARSER}"         \
                    --from-literal=SEARCH_SERVICENAME="${ES_SERVICENAME}"                 \
                    --from-literal=MON_NS="${MON_NS}"
 
-kubectl -n $LOG_NS label configmap fb-env-vars   managed-by=v4m-es-script
+kubectl -n "$LOG_NS" label configmap fb-env-vars   managed-by=v4m-es-script
 
 # Check to see if we are upgrading from earlier version requiring root access
 if [ "$( kubectl -n $LOG_NS get configmap fb-dbmigrate-script -o name --ignore-not-found)" != "configmap/fb-dbmigrate-script" ]; then
@@ -167,9 +167,9 @@ if [ "$( kubectl -n $LOG_NS get configmap fb-dbmigrate-script -o name --ignore-n
 fi
 
 # Create ConfigMap containing Fluent Bit database migration script
-kubectl -n $LOG_NS delete configmap fb-dbmigrate-script --ignore-not-found
-kubectl -n $LOG_NS create configmap fb-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
-kubectl -n $LOG_NS label  configmap fb-dbmigrate-script managed-by=v4m-es-script
+kubectl -n "$LOG_NS" delete configmap fb-dbmigrate-script --ignore-not-found
+kubectl -n "$LOG_NS" create configmap fb-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n "$LOG_NS" label  configmap fb-dbmigrate-script managed-by=v4m-es-script
 
 ## Get Helm Chart Name
 log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
@@ -189,6 +189,7 @@ helm $helmDebug upgrade --install --namespace $LOG_NS v4m-fb  \
   $chart2install
 
 #pause to allow migration script to complete (if necessary)
+log_debug "Pausing to allow migration script to complete"
 sleep 20
 
 #Container Security: Disable Token Automounting at ServiceAccount; enable for Pod

From 0cb3bcfa77250397edeb56678d0e711a9075a94f Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Thu, 19 Dec 2024 16:51:14 -0500
Subject: [PATCH 29/34] Cleanup from code review (2)

---
 logging/fb/fb_initcontainer_image.template | 1 -
 1 file changed, 1 deletion(-)

diff --git a/logging/fb/fb_initcontainer_image.template b/logging/fb/fb_initcontainer_image.template
index 4d4b69c0..359f1f72 100644
--- a/logging/fb/fb_initcontainer_image.template
+++ b/logging/fb/fb_initcontainer_image.template
@@ -1,7 +1,6 @@
 initContainers:
 - name: chowner-v4m-fb-storage
   image: __IMAGE_REPO_3LEVEL__:__IMAGE_TAG__
-  ##image: docker.io/library/busybox:latest
   imagePullPolicy: IfNotPresent
   command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
   securityContext:

From 0488a174799c6f893d9aa938f04926a8ef84de1c Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Fri, 20 Dec 2024 15:17:32 -0500
Subject: [PATCH 30/34] OpenSearch pods: readOnlyRootFilesystem set to 'true'

---
 CHANGELOG.md                                  |  9 +++++
 logging/bin/deploy_opensearch.sh              |  1 +
 .../opensearch/opensearch_helm_values.yaml    | 33 ++++++++++++++++++-
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c2b3f39..2ce9669c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # SAS Viya Monitoring for Kubernetes
 
+## Unreleased
+* **Logging**
+  * [SECURITY] OpenSearch pods has been reconfigured to allow `readOnlyRootFilesystem` to be set to 'true'. A
+new initContainer has been added to facilitate this.
+  * [SECURITY] Other runtime security controls for OpenSearch pods have also been tightened.  Changes include: 
+disallowing privileged containers, privilege escalation and removing all Linux capabilities. NOTE: some
+initContainers continue to run as `root` user but only run briefly during the initial deployment process.
+
+
 ## Version 1.2.32 (09DEC2024)
 * **Overall**
   * [CHANGE] Comments added to user.env files within samples/generic-base to clarify security best-practices; other
diff --git a/logging/bin/deploy_opensearch.sh b/logging/bin/deploy_opensearch.sh
index 004e9c32..b3f40626 100755
--- a/logging/bin/deploy_opensearch.sh
+++ b/logging/bin/deploy_opensearch.sh
@@ -40,6 +40,7 @@ fi
 #Generate yaml files with all container-related keys
 generateImageKeysFile "$OS_FULL_IMAGE"          "logging/opensearch/os_container_image.template"
 generateImageKeysFile "$OS_SYSCTL_FULL_IMAGE"   "$imageKeysFile"  "OS_SYSCTL_"
+generateImageKeysFile "$OS_FULL_IMAGE"          "logging/opensearch/os_initcontainer_image.template"
 
 # get credentials
 export ES_ADMIN_PASSWD=${ES_ADMIN_PASSWD}
diff --git a/logging/opensearch/opensearch_helm_values.yaml b/logging/opensearch/opensearch_helm_values.yaml
index 6bf2b0f4..0447be18 100644
--- a/logging/opensearch/opensearch_helm_values.yaml
+++ b/logging/opensearch/opensearch_helm_values.yaml
@@ -94,6 +94,12 @@ extraVolumes:
 - name: admin-certs
   secret:
     secretName: es-admin-tls-secret
+- name: tmpspace
+  emptyDir: {}
+- name: logspace
+  emptyDir: {}
+- name: configdir
+  emptyDir: {}
 
 extraVolumeMounts: 
 - mountPath: "/usr/share/opensearch/config/run_securityadmin.sh"
@@ -128,7 +134,12 @@ extraVolumeMounts:
 - name: admin-certs
   mountPath: "/usr/share/opensearch/config/admin-root-ca.pem"
   subPath: ca.crt
-
+- name: tmpspace
+  mountPath: "/tmp"
+- name: logspace
+  mountPath: "/usr/share/opensearch/logs"
+- name: configdir
+  mountPath: /usr/share/opensearch/config/
 
 #09FEB22 TBD: affinity.podAntiAffinity ==> nodeAffinity?
 roles:
@@ -178,5 +189,25 @@ sysctlInit:
 
 
 podSecurityContext:
+  ###runAsUser: 1000     #from Helm chart 
   seccompProfile: 
     type: RuntimeDefault
+
+
+securityContext:
+  privileged: false
+  readOnlyRootFilesystem: true
+  allowPrivilegeEscalation: false
+  ###runAsNonRoot: true  #from Helm chart
+  ###runAsUser: 1000     #from Helm chart 
+  ###  capabilities:     #from Helm chart
+  ###  drop: ["ALL"]     #from Helm chart
+
+
+
+##
+## NOTE: An initContainer is also deployed; it is defined
+##       in logging/opensearch/os_initcontainer_image.template
+##       which is passed to the Helm install command
+##       after value substitution.
+##

From 41dfff14eedda4f1007cc1971a4274f90fe675cb Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Mon, 23 Dec 2024 11:46:20 -0500
Subject: [PATCH 31/34] Tighten container security: FB - Event collection

---
 logging/fb/fluent-bit_helm_values_events.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/logging/fb/fluent-bit_helm_values_events.yaml b/logging/fb/fluent-bit_helm_values_events.yaml
index 5a009db2..a82e8135 100644
--- a/logging/fb/fluent-bit_helm_values_events.yaml
+++ b/logging/fb/fluent-bit_helm_values_events.yaml
@@ -39,6 +39,8 @@ securityContext:
   runAsNonRoot: true
   runAsUser: 1001
   readOnlyRootFilesystem: true
+  capabilities:
+    drop: ["ALL"]
 
 podSecurityContext:
   seccompProfile:

From 855dcc12d5f5fd788dd2360e8f0828c76ae4342c Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Mon, 23 Dec 2024 11:51:49 -0500
Subject: [PATCH 32/34] OpenSearch pods: set 'securityContext.privileged' to
 'false'

---
 logging/opensearch/osd_helm_values.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/logging/opensearch/osd_helm_values.yaml b/logging/opensearch/osd_helm_values.yaml
index 3c823609..aafc270e 100644
--- a/logging/opensearch/osd_helm_values.yaml
+++ b/logging/opensearch/osd_helm_values.yaml
@@ -69,6 +69,7 @@ config:
 securityContext:
   readOnlyRootFilesystem: true
   allowPrivilegeEscalation: false
+  privileged: false
 
 podSecurityContext:
   seccompProfile: 

From 9f2b20bb9338b4c0ac17aed8db6be9746c99b1ad Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Mon, 23 Dec 2024 11:57:54 -0500
Subject: [PATCH 33/34] Tighten container security: ES Exporter

---
 logging/esexporter/values-es-exporter.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/logging/esexporter/values-es-exporter.yaml b/logging/esexporter/values-es-exporter.yaml
index b3b7e8aa..58145d08 100644
--- a/logging/esexporter/values-es-exporter.yaml
+++ b/logging/esexporter/values-es-exporter.yaml
@@ -173,3 +173,6 @@ prometheusRule:
     annotations:
       description: The heap usage is over 90% for 15m
       summary: Elasticsearch node {{$labels.node}} heap usage is high
+
+securityContext:
+  privileged: false

From 20ea081ac97054bd9308f936069c3de121872492 Mon Sep 17 00:00:00 2001
From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com>
Date: Mon, 23 Dec 2024 14:23:05 -0500
Subject: [PATCH 34/34] Add initContainer definition

---
 .../os_initcontainer_image.template           | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 logging/opensearch/os_initcontainer_image.template

diff --git a/logging/opensearch/os_initcontainer_image.template b/logging/opensearch/os_initcontainer_image.template
new file mode 100644
index 00000000..3d598ba6
--- /dev/null
+++ b/logging/opensearch/os_initcontainer_image.template
@@ -0,0 +1,21 @@
+extraInitContainers:
+  - name: copy-conf-data
+##    image: opensearchproject/opensearch:2.17.1
+    image: __IMAGE_REPO_3LEVEL__:__IMAGE_TAG__
+    imagePullPolicy: __IMAGE_PULL_POLICY__
+    command:
+    - sh
+    - -c
+    - cp -r /usr/share/opensearch/config/* /config/
+    - chmod -R 777 /config/
+    - ls /config/
+    volumeMounts:
+      - name: configdir
+        mountPath: /config/
+    securityContext:
+      allowPrivilegeEscalation: false
+      privileged: false
+      readOnlyRootFilesystem: true
+      runAsNonRoot: true
+      capabilities:
+        drop: ["ALL"]