Fbnoroot2

.github/workflows/build-artifact/ARTIFACT_INVENTORY.template

@@ -16,9 +16,9 @@ registry/repository/image_name:version
 
 | Subsystem| Component | Fully Qualified Container-Image Name (registry/repository/image_name:version)|
 |----|----|----|
-| Logging | BusyBox (OpenSearch) | __OS_SYSCTL_FULL_IMAGE__ |
 | Logging | Fluent Bit | __FB_FULL_IMAGE__ |
 | Logging | Elasticsearch Exporter | __ES_EXPORTER_FULL_IMAGE__ |
+| Logging | initContainer (Fluent Bit, OpenSearch) | __OS_SYSCTL_FULL_IMAGE__ |
 | Logging | OpenSearch | __OS_FULL_IMAGE__ |
 | Logging | OpenSearch Dashboards| __OSD_FULL_IMAGE__ |
 | Metrics | Alertmanager | __ALERTMANAGER_FULL_IMAGE__ |

CHANGELOG.md

@@ -1,5 +1,18 @@
 # SAS Viya Monitoring for Kubernetes
 
+## Unreleased
+* **Logging**
+  * [SECURITY] Fluent Bit log collecting pods no longer run as `root` user.  In addition, the database used to
+maintain state information for the log collector has moved to a hostPath volume and been renamed. A new initContainer
+has been added to handle migrating any existing state information and make adjustments to file ownership/permissions.
+NOTE: This initContainer runs under as `root` user but only runs briefly during the initial deployment process.
+  * [SECURITY] Runtime security controls for all Fluent Bit pods (inc. both log collecting and Kubernetes event
+collecting) have been tightened.  Changes include: adding seecompProfile; and disallowing privileged containers,
+privilege escalation and write access to the root filesystem.
+  * [SECURITY] On OpenShift, all Fluent Bit pods now use custom SCC objects to support changes described above.
+  * [CHANGE] Improved handling of long log messages and those from some Crunchy Data pods
+
+
 ## Version 1.2.32 (09DEC2024)
 * **Overall**
   * [CHANGE] Comments added to user.env files within samples/generic-base to clarify security best-practices; other
@@ -8,6 +21,7 @@ cleanup.
   * [SECURITY] Set `seccompProfile` to `RuntimeDefault` for OpenSearch, OpenSearch Dashboards and Fluent Bit pods in
 non-OpenShift environments.
 
+
 ## Version 1.2.31 (15NOV2024)
 * **Logging**
   * [UPGRADE] OpenSearch and OpenSearch Dashboards upgraded from 2.15.0 to 2.17.1
@@ -16,6 +30,7 @@ required a new serviceMonitor (elasticsearch-v2) be deployed.
   * [UPGRADE] Fluent Bit upgraded from 3.1.3 to 3.1.9
   * [UPGRADE] OpenSearch Data Source Plugin to Grafana upgraded from 2.18.0 to 2.21.1
 
+
 ## Version 1.2.30 (11OCT2024)
 * **Logging**
   * [SECURITY] OpenSearch Dashboards pod `securityContext` updated to set allowPrivilegeEscalation to 'false'

component_versions.env

@@ -19,6 +19,7 @@ FLUENTBIT_HELM_CHART_REPO=fluent
 FLUENTBIT_HELM_CHART_NAME=fluent-bit
 FLUENTBIT_HELM_CHART_VERSION=0.47.10
 FB_FULL_IMAGE="cr.fluentbit.io/fluent/fluent-bit:3.1.9"
+FB_INITCONTAINER_FULL_IMAGE="docker.io/library/busybox:latest"
 
 #OpenSearch
 OPENSEARCH_HELM_CHART_REPO=opensearch

logging/bin/deploy_fluentbit_azmonitor.sh

@@ -39,7 +39,8 @@ fi
 log_info "Deploying Fluent Bit (Azure Monitor)"
 
 #Generate yaml file with all container-related keys#Generate yaml file with all container-related keys
-generateImageKeysFile "$FB_FULL_IMAGE"          "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_FULL_IMAGE"               "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_INITCONTAINER_FULL_IMAGE" "logging/fb/fb_initcontainer_image.template"
 
 # Fluent Bit user customizations
 FB_AZMONITOR_USER_YAML="${FB_AZMONITOR_USER_YAML:-$USER_DIR/logging/user-values-fluent-bit-azmonitor.yaml}"
@@ -94,7 +95,7 @@ else
 fi
 
 # Multiline parser setup
-LOG_MULTILINE_ENABLED="${LOG_MULTILINE_ENABLED}"
+LOG_MULTILINE_ENABLED="${LOG_MULTILINE_ENABLED:-true}"
 if [ "$LOG_MULTILINE_ENABLED" == "true" ]; then
   LOG_MULTILINE_PARSER="docker, cri"
 else
@@ -154,6 +155,18 @@ kubectl -n $LOG_NS create configmap fbaz-env-vars \
 
 kubectl -n $LOG_NS label configmap fbaz-env-vars   managed-by=v4m-es-script
 
+# Check to see if we are upgrading from earlier version requiring root access
+if [ "$( kubectl -n $LOG_NS get configmap fbaz-dbmigrate-script -o name --ignore-not-found)" != "configmap/fbaz-dbmigrate-script" ]; then
+   log_debug "Removing FB pods (if they exist) to allow migration."
+   kubectl -n "$LOG_NS" delete daemonset v4m-fbaz --ignore-not-found
+fi
+
+# Create ConfigMap containing Fluent Bit database migration script
+kubectl -n $LOG_NS delete configmap fbaz-dbmigrate-script --ignore-not-found
+kubectl -n $LOG_NS create configmap fbaz-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n $LOG_NS label  configmap fbaz-dbmigrate-script managed-by=v4m-es-script
+
+
  ## Get Helm Chart Name
  log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
  chart2install="$(get_helmchart_reference $FLUENTBIT_HELM_CHART_REPO $FLUENTBIT_HELM_CHART_NAME $FLUENTBIT_HELM_CHART_VERSION)"
@@ -170,9 +183,12 @@ helm $helmDebug upgrade --install v4m-fbaz  --namespace $LOG_NS  \
   --set fullnameOverride=v4m-fbaz \
   $chart2install
 
+#pause to allow migration script to complete (if necessary)
+sleep 20
 
 #Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
 disable_sa_token_automount $LOG_NS v4m-fbaz
+# FB pods will restart after following call if automount is not already enabled
 enable_pod_token_automount $LOG_NS daemonset v4m-fbaz
 
 # Force restart of daemonset to ensure we pick up latest config changes

logging/bin/deploy_fluentbit_opensearch.sh

@@ -68,7 +68,8 @@ else
 fi
 
 #Generate yaml file with all container-related keys
-generateImageKeysFile "$FB_FULL_IMAGE"          "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_FULL_IMAGE"                "logging/fb/fb_container_image.template"
+generateImageKeysFile "$FB_INITCONTAINER_FULL_IMAGE"  "logging/fb/fb_initcontainer_image.template"
 
 # Fluent Bit user customizations
 FB_OPENSEARCH_USER_YAML="${FB_OPENSEARCH_USER_YAML:-$USER_DIR/logging/user-values-fluent-bit-opensearch.yaml}"
@@ -98,7 +99,7 @@ fi
 log_debug "Using FB ConfigMap:" $FB_CONFIGMAP
 
 # Multiline parser setup
-LOG_MULTILINE_ENABLED=${LOG_MULTILINE_ENABLED}
+LOG_MULTILINE_ENABLED=${LOG_MULTILINE_ENABLED:-true}
 if [ "$LOG_MULTILINE_ENABLED" == "true" ]; then
   LOG_MULTILINE_PARSER="docker, cri"
 else
@@ -159,6 +160,17 @@ kubectl -n $LOG_NS create configmap fb-env-vars \
 
 kubectl -n $LOG_NS label configmap fb-env-vars   managed-by=v4m-es-script
 
+# Check to see if we are upgrading from earlier version requiring root access
+if [ "$( kubectl -n $LOG_NS get configmap fb-dbmigrate-script -o name --ignore-not-found)" != "configmap/fb-dbmigrate-script" ]; then
+   log_debug "Removing FB pods (if they exist) to allow migration."
+   kubectl -n "$LOG_NS" delete daemonset v4m-fb --ignore-not-found
+fi
+
+# Create ConfigMap containing Fluent Bit database migration script
+kubectl -n $LOG_NS delete configmap fb-dbmigrate-script --ignore-not-found
+kubectl -n $LOG_NS create configmap fb-dbmigrate-script --from-file logging/fb/migrate_fbstate_db.sh
+kubectl -n $LOG_NS label  configmap fb-dbmigrate-script managed-by=v4m-es-script
+
 ## Get Helm Chart Name
 log_debug "Fluent Bit Helm Chart: repo [$FLUENTBIT_HELM_CHART_REPO] name [$FLUENTBIT_HELM_CHART_NAME] version [$FLUENTBIT_HELM_CHART_VERSION]"
 chart2install="$(get_helmchart_reference $FLUENTBIT_HELM_CHART_REPO $FLUENTBIT_HELM_CHART_NAME $FLUENTBIT_HELM_CHART_VERSION)"
@@ -176,8 +188,12 @@ helm $helmDebug upgrade --install --namespace $LOG_NS v4m-fb  \
   --set fullnameOverride=v4m-fb \
   $chart2install
 
+#pause to allow migration script to complete (if necessary)
+sleep 20
+
 #Container Security: Disable Token Automounting at ServiceAccount; enable for Pod
 disable_sa_token_automount $LOG_NS v4m-fb
+# FB pods will restart after following call if automount is not already enabled
 enable_pod_token_automount $LOG_NS daemonset v4m-fb
 
 # Force restart of daemonset to ensure we pick up latest config changes

logging/bin/deploy_openshift_prereqs.sh

@@ -25,6 +25,20 @@ fi
 # link OpenSearch serviceAccounts to 'privileged' scc
 oc adm policy add-scc-to-user privileged -z v4m-os -n $LOG_NS
 
+# create the 'v4m-logging-v2' SCC, if it does not already exist
+if oc get scc v4m-logging-v2 2>/dev/null 1>&2; then
+   log_info "Skipping scc creation; using existing scc [v4m-logging-v2]"
+else
+   oc create -f logging/openshift/fb_v4m-logging-v2_scc.yaml
+fi
+
+# create the 'v4m-k8sevents' SCC, if it does not already exist
+if oc get scc v4m-k8sevents 2>/dev/null 1>&2; then
+   log_info "Skipping scc creation; using existing scc [v4m-k8sevents]"
+else
+   oc create -f logging/openshift/fb_v4m-k8sevents_scc.yaml
+fi
+
 log_info "OpenShift Prerequisites have been deployed."
 
 log_debug "Script [$this_script] has completed [$(date)]"

logging/bin/remove_fluentbit_azmonitor.sh

@@ -30,7 +30,7 @@ kubectl -n $LOG_NS delete configmap fbaz-fluent-bit-config   --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-viya-parsers        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-viya-tracing        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fbaz-env-vars            --ignore-not-found
-
+kubectl -n $LOG_NS delete configmap fbaz-dbmigrate-script    --ignore-not-found
 
 # Should we leave secret in place?
 log_info "Removing Connection information  (secret)"

logging/bin/remove_fluentbit_opensearch.sh

@@ -27,6 +27,7 @@ kubectl -n $LOG_NS delete configmap fb-fluent-bit-config   --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-viya-parsers        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-viya-tracing        --ignore-not-found
 kubectl -n $LOG_NS delete configmap fb-env-vars            --ignore-not-found
+kubectl -n $LOG_NS delete configmap fb-dbmigrate-script    --ignore-not-found
 
 log_debug "Script [$this_script] has completed [$(date)]"
 echo ""

logging/bin/remove_openshift_artifacts.sh

@@ -19,7 +19,10 @@ if [ "$OPENSHIFT_ARTIFACTS_REMOVE" != "true" ]; then
 fi
 
 # remove custom OpenShift SCC
-oc delete scc v4mlogging --ignore-not-found
+oc delete scc v4mlogging     --ignore-not-found
+oc delete scc v4m-logging-v2 --ignore-not-found
+oc delete scc v4m-k8sevents  --ignore-not-found
+
 
 
 log_info "OpenShift Prerequisites have been removed."

logging/fb/fb_initcontainer_image.template

@@ -0,0 +1,27 @@
+initContainers:
+- name: chowner-v4m-fb-storage
+  image: __IMAGE_REPO_3LEVEL__:__IMAGE_TAG__
+  ##image: docker.io/library/busybox:latest
+  imagePullPolicy: IfNotPresent
+  command: ['sh', '-c', "./usr/bin/migrate_fbstate_db.sh"]
+  securityContext:
+    privileged: true
+    allowPrivilegeEscalation: true
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop: ["all"]
+      add: ["CHOWN"]
+    runAsUser: 0
+    runAsNonRoot: false
+  volumeMounts:
+  - name: v4m-fb-storage
+    mountPath: /var/log/v4m-fb-storage
+  - name: dbmigrate-script
+    mountPath: /usr/bin/migrate_fbstate_db.sh
+    readOnly: false
+    subPath: migrate_fbstate_db.sh
+  - mountPath: /var/log
+    name: varlog
+    readOnly: true
+
+

logging/fb/fluent-bit_config.configmap_azmonitor.yaml

@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.26
+       Set  fb_configMap_version 0.2.27noroot2a
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -363,6 +363,7 @@ data:
        Match           kube.*
        Key_name        collectapi_message
        Parser          crunchy_poo
+       Parser          crunchy_exporter
        Parser          collectapi
        Parser          collectapiT
        Parser          crunchy_green
@@ -972,6 +973,7 @@ data:
        Allowlist_key  properties
     [INPUT]
        Name             tail
+       Alias            tail4logs
        Path             /var/log/containers/*.log
        Exclude_Path     /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
@@ -982,12 +984,13 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [INPUT]
        # NOTE: ** METALOGGING **
        Name             tail
+       Alias            tail4ops
        Path             /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
        Tag              opsK.*
@@ -996,7 +999,7 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [OUTPUT]
@@ -1015,7 +1018,7 @@ data:
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
-       storage.path /tmp/fb_buffer/
+       storage.path /var/log/v4m-fb-storage
        storage.checksum off
        storage.sync normal
        storage.backlog.mem_limit 5M

logging/fb/fluent-bit_config.configmap_opensearch.yaml

@@ -6,7 +6,7 @@ data:
     [FILTER]
        Name modify
        Match *
-       Set  fb_configMap_version 0.2.26
+       Set  fb_configMap_version 0.2.27noroot2a
        #Set clusterID NOT_SET
        # initialized to N to force level standardization
        Set  __temp_level_fixed   N
@@ -363,6 +363,7 @@ data:
        Match           kube.*
        Key_name        collectapi_message
        Parser          crunchy_poo
+       Parser          crunchy_exporter
        Parser          collectapi
        Parser          collectapiT
        Parser          crunchy_green
@@ -974,6 +975,7 @@ data:
        Allowlist_key  properties
     [INPUT]
        Name             tail
+       Alias            tail4logs
        Path             /var/log/containers/*.log
        Exclude_Path     /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
@@ -984,12 +986,13 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [INPUT]
        # NOTE: ** METALOGGING **
        Name             tail
+       Alias            tail4ops
        Path             /var/log/containers/v4m-*.log
        Parser           ${KUBERNETES_RUNTIME_LOGFMT}
        Tag              opsK.*
@@ -998,7 +1001,7 @@ data:
        Skip_Long_Lines  On
        Read_from_Head   On
        Ignore_Older     1d
-       DB               /var/log/sas_viya_flb.db
+       DB               /var/log/v4m-fb-storage/v4m_fb.db
        DB.locking       On
        storage.type     filesystem
     [OUTPUT]
@@ -1053,7 +1056,7 @@ data:
        HTTP_Server  On
        HTTP_Listen  0.0.0.0
        HTTP_Port    2020
-       storage.path /tmp/fb_buffer/
+       storage.path /var/log/v4m-fb-storage
        storage.checksum off
        storage.sync normal
        storage.backlog.mem_limit 5M

logging/fb/fluent-bit_helm_values_azmonitor.yaml

@@ -31,7 +31,14 @@ extraVolumeMounts:
 - mountPath: /fluent-bit/etc/viya-tracing.conf
   name: tracing-config
   subPath: viya-tracing.conf
+- mountPath: /var/log/v4m-fb-storage
+  name: v4m-fb-storage
+  readOnly: false
 extraVolumes:
+- hostPath:
+    path: /var/log/v4m-fb-storage
+    type: DirectoryOrCreate
+  name: v4m-fb-storage
 - configMap:
     defaultMode: 420
     name: fbaz-viya-parsers
@@ -40,6 +47,10 @@ extraVolumes:
     defaultMode: 420
     name: fbaz-viya-tracing
   name: tracing-config
+- configMap:
+    defaultMode: 0755
+    name: fbaz-dbmigrate-script
+  name: dbmigrate-script
 fullConfigMap: false
 existingConfigMap: fbaz-fluent-bit-config
 
@@ -70,6 +81,22 @@ resources:
     cpu: 100m
     memory: 128Mi
 
+securityContext:
+  runAsUser: 3301
+  readOnlyRootFilesystem: true
+  privileged: false
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop: ["ALL"]
+
 podSecurityContext:
+  runAsNonRoot: true
   seccompProfile:
     type: RuntimeDefault
+
+##
+## NOTE: An initContainer is also deployed; it is defined
+##       in logging/fb/fb_initcontainer_image.template
+##       which is passed to the Helm install command
+##       after value substitution.
+##