Revamp volume configuration, fix issues introduced in 2.4.0 (#218)

* use emptydir volumes instead of hostpath defaults to avoid issues with readonly hosts
* set default path for hostPath if nothing is set
* make the chmod init container optional, fix the permission issues using fsgroup (see below)
* add missing runAsGroup configuration to container securitypolicy
* enable fsGroup 101 (fluentd) by default to get rid of the volume mount hack. the container will get 101 as a supplemental group and will be allowed to write on the volume
* rbac fixes for podsecurity policy roles and proper use of service account name
* generic KubernetesStorage implementation and use it for fluentd as well

Author: pepov

config/crd/bases/logging.banzaicloud.io_loggings.yaml

@@ -8,4 +8,3 @@ spec:
   }
   fluentbit: {}
   controlNamespace: default
-

config/samples/logging_v1alpha2_logging_default.yaml

@@ -0,0 +1,12 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Logging
+metadata:
+  name: defaultlogging
+spec:
+  fluentd:
+    security:
+      podSecurityPolicyCreate: true
+  fluentbit:
+    security:
+      podSecurityPolicyCreate: true
+  controlNamespace: default

config/samples/logging_v1alpha2_logging_psp.yaml

@@ -0,0 +1,8 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Logging
+metadata:
+  name: defaultlogging
+spec:
+  fluentd: {}
+  fluentbit: {}
+  controlNamespace: default

config/samples/logging_v1alpha2_logging_pvc.yaml

@@ -125,11 +125,11 @@ spec:
 | tolerations | [Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core) | {} | Pod toleration |
 | metrics | [Metrics](./logging-operator-monitoring.md#metrics-variables) | {} | Metrics defines the service monitor endpoints |
 | security | [Security](./security#security-variables) | {} | Security defines Fluentd, Fluentbit deployment security properties |
-| position_db |  [KubernetesStorage](#KubernetesStorage) | nil | Add position db storage support. If nothing is configured a `hostPath` volume is used with the path `/opt/fluent-bit/<name of the logging CR>/pos`. |
+| position_db |  [KubernetesStorage](#KubernetesStorage) | nil | Add position db storage support. If nothing is configured an emptyDir volume will be used. |
 | inputTail | [InputTail](./fluentbit.md#tail-inputtail) | {} | The tail input plugin allows to monitor one or several text files.  |
 | filterKubernetes | [FilterKubernetes](./fluentbit.md#kubernetes-filterkubernetes) | {} | Fluent Bit Kubernetes Filter allows to enrich your log files with Kubernetes metadata. |
 | bufferStorage | [BufferStorage](./fluentbit.md#bufferstorage) |  | Buffer Storage configures persistent buffer to avoid losing data in case of a failure |
-| bufferStorageVolume | [KubernetesStorage](#KubernetesStorage) | nil | Volume definition for the Buffer Storage. If nothing is configured a `hostPath` volume is used with the path `/opt/fluent-bit/<name of the logging CR>/buf`. |
+| bufferStorageVolume | [KubernetesStorage](#KubernetesStorage) | nil | Volume definition for the Buffer Storage. If nothing is configured an emptydir volume will be used. |
 | customConfigSecret | string | "" | Custom secret to use as fluent-bit config.<br /> It must include all the config files necessary to run fluent-bit (_fluent-bit.conf_, _parsers*.conf_) |
 
 **`logging` with custom fluent-bit annotations** 
@@ -214,10 +214,20 @@ Define Kubernetes storage
 
 | Name      | Type | Default | Description |
 |-----------|------|---------|-------------|
-| host_path | [HostPathVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#hostpathvolumesource-v1-core) | - | Represents a host path mapped into a pod. |
+| host_path | [HostPathVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#hostpathvolumesource-v1-core) | - | Represents a host path mapped into a pod. If path is empty, it will automatically be set to "/opt/logging-operator/<name of the logging CR>/<name of the volume>" |
+| emptyDir | [EmptyDirVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#emptydirvolumesource-v1-core) | - | Represents an empty directory for a pod. |
+| pvc | [PersistentVolumeClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#persistentvolumeclaim-v1-core) | - | A PersistentVolumeClaim (PVC) is a request for storage by a user. |
 
+#### Persistent Volume Claim
 
-## outputs, clusteroutputs
+| Name      | Type | Default | Description |
+|-----------|------|---------|-------------|
+| spec | [PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#persistentvolumeclaimspec-v1-core) | - | Spec defines the desired characteristics of a volume requested by a pod author. |
+| source | [PersistentVolumeClaimVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#persistentvolumeclaimvolumesource-v1-core) | - | PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace.  |
+
+The Persistent Volume Claim should be created with the given `spec` and with the `name` defined in the `source`'s `claimName`.
+
+## Outputs, Clusteroutputs
 
 Outputs are the final stage for a `logging flow`. You can define multiple `outputs` and attach them to multiple `flows`.
 

docs/crds.md

@@ -103,8 +103,6 @@ helm install --namespace logging --name nginx-demo banzaicloud-stable/nginx-logg
   - kind: ServiceAccount
     name: nginx-demo-nginx-logging-demo-logging-fluentd
     namespace: logging
-
-
 ```
 
 #### Fluentbit ClusterRole & ClusterRoleBinding Output
@@ -212,19 +210,23 @@ helm install --namespace logging --name nginx-demo banzaicloud-stable/nginx-logg
 
 #### Fluentd PSP+Role Output
 ``` 
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: nginx-demo-nginx-logging-demo-logging-fluentd-psp
-  namespace: logging
 rules:
 - apiGroups:
   - policy
+  - extensions
   resources:
+  - podsecuritypolicies
+  resourceNames:
   - nginx-demo-nginx-logging-demo-logging-fluentd
   verbs:
   - use
 
 ---
+apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:
   name: nginx-demo-nginx-logging-demo-logging-fluentd
@@ -251,12 +253,14 @@ spec:
   - configMap
   - emptyDir
   - secret
-  - persistentVolumeClaim
+  - hostPath
+
 
 
 ```
 #### Fluentbit PSP+ClusterRole Output
 ```
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: nginx-demo-nginx-logging-demo-logging-fluentbit-psp
@@ -268,6 +272,7 @@ rules:
   verbs:
   - use
 ---
+apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:
   name: nginx-demo-nginx-logging-demo-logging-fluentbit
@@ -323,14 +328,14 @@ spec:
         allowPrivilegeEscalation: false
         readOnlyRootFilesystem: false
       podSecurityContext:
-        fsGroup: 65533
+        fsGroup: 101
   fluentbit:
     security:
       securityContext:
         allowPrivilegeEscalation: false
         readOnlyRootFilesystem: true
       podSecurityContext:
-        fsGroup: 65533
+        fsGroup: 101
   controlNamespace: logging
 EOF
 ```
@@ -339,10 +344,10 @@ EOF
 helm install --namespace logging --name nginx-demo banzaicloud-stable/nginx-logging-demo \
     --set=loggingOperator.fluentd.security.securityContext.allowPrivilegeEscalation=False \
     --set=loggingOperator.fluentd.security.securityContext.readOnlyRootFilesystem=False \
-    --set=loggingOperator.fluentd.security.podSecurityContext.fsGroup=65533 \
+    --set=loggingOperator.fluentd.security.podSecurityContext.fsGroup=101 \
     --set=loggingOperator.fluentbit.security.securityContext.allowPrivilegeEscalation=False \
     --set=loggingOperator.fluentbit.security.securityContext.readOnlyRootFilesystem=True \
-    --set=loggingOperator.fluentbit.security.podSecurityContext.fsGroup=65533
+    --set=loggingOperator.fluentbit.security.podSecurityContext.fsGroup=101
 ```
 
 ### Example Manifest Generated by the operator
@@ -364,7 +369,7 @@ spec:
 ...
   schedulerName: default-scheduler
   securityContext:
-    fsGroup: 65533
+    fsGroup: 101
   serviceAccount: nginx-demo-nginx-logging-demo-logging-fluentd
 ...
 ```

docs/security/README.md

@@ -20,7 +20,6 @@ import (
 
 	"github.com/banzaicloud/logging-operator/pkg/k8sutil"
 	"github.com/banzaicloud/logging-operator/pkg/resources/templates"
-	"github.com/banzaicloud/logging-operator/pkg/sdk/api/v1beta1"
 	"github.com/banzaicloud/logging-operator/pkg/sdk/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -197,25 +196,7 @@ func (r *Reconciler) generateVolume() (v []corev1.Volume) {
 		}
 		v = append(v, tlsRelatedVolume)
 	}
-	v = append(v, GetVolumeFromKubernetesStorage(r.Logging.Spec.FluentbitSpec.PositionDB, TailPositionVolume))
-	v = append(v, GetVolumeFromKubernetesStorage(r.Logging.Spec.FluentbitSpec.BufferStorageVolume, BufferStorageVolume))
+	v = append(v, r.Logging.Spec.FluentbitSpec.PositionDB.GetVolume(r.Logging.Name, TailPositionVolume))
+	v = append(v, r.Logging.Spec.FluentbitSpec.BufferStorageVolume.GetVolume(r.Logging.Name, BufferStorageVolume))
 	return
 }
-
-func GetVolumeFromKubernetesStorage(storage *v1beta1.KubernetesStorage, name string) corev1.Volume {
-	volume := corev1.Volume{
-		Name: name,
-	}
-	if storage != nil {
-		if storage.HostPath != nil {
-			volume.VolumeSource = corev1.VolumeSource{
-				HostPath: storage.HostPath,
-			}
-		}
-	} else {
-		volume.VolumeSource = corev1.VolumeSource{
-			EmptyDir: &corev1.EmptyDirVolumeSource{},
-		}
-	}
-	return volume
-}

pkg/resources/fluentbit/daemonset.go

@@ -80,9 +80,10 @@ func (r *Reconciler) pspClusterRole() (runtime.Object, k8sutil.DesiredState) {
 				r.Logging.QualifiedName(clusterRoleName+"-psp"), r.Logging.Labels, r.Logging),
 			Rules: []rbacv1.PolicyRule{
 				{
-					APIGroups: []string{"policy"},
-					Resources: []string{r.Logging.QualifiedName(fluentbitPodSecurityPolicyName)},
-					Verbs:     []string{"use"},
+					APIGroups:     []string{"policy"},
+					Resources:     []string{"podsecuritypolicies"},
+					ResourceNames: []string{r.Logging.QualifiedName(fluentbitPodSecurityPolicyName)},
+					Verbs:         []string{"use"},
 				},
 			},
 		}, k8sutil.StatePresent
@@ -108,7 +109,7 @@ func (r *Reconciler) pspClusterRoleBinding() (runtime.Object, k8sutil.DesiredSta
 			Subjects: []rbacv1.Subject{
 				{
 					Kind:      "ServiceAccount",
-					Name:      r.Logging.QualifiedName(defaultServiceAccountName),
+					Name:      r.getServiceAccount(),
 					Namespace: r.Logging.Spec.ControlNamespace,
 				},
 			},

pkg/resources/fluentbit/psp.go

@@ -35,6 +35,7 @@ func (r *Reconciler) clusterPodSecurityPolicy() (runtime.Object, k8sutil.Desired
 					"configMap",
 					"emptyDir",
 					"secret",
+					"hostPath",
 					"persistentVolumeClaim"},
 				SELinux: policyv1beta1.SELinuxStrategyOptions{
 					Rule: policyv1beta1.SELinuxStrategyRunAsAny,
@@ -68,9 +69,10 @@ func (r *Reconciler) pspRole() (runtime.Object, k8sutil.DesiredState) {
 			ObjectMeta: templates.FluentdObjectMeta(r.Logging.QualifiedName(roleName+"-psp"), r.Logging.Labels, r.Logging),
 			Rules: []rbacv1.PolicyRule{
 				{
-					APIGroups: []string{"policy"},
-					Resources: []string{r.Logging.QualifiedName(PodSecurityPolicyName)},
-					Verbs:     []string{"use"},
+					APIGroups:     []string{"policy"},
+					Resources:     []string{"podsecuritypolicies"},
+					ResourceNames: []string{r.Logging.QualifiedName(PodSecurityPolicyName)},
+					Verbs:         []string{"use"},
 				},
 			},
 		}, k8sutil.StatePresent
@@ -95,7 +97,7 @@ func (r *Reconciler) pspRoleBinding() (runtime.Object, k8sutil.DesiredState) {
 			Subjects: []rbacv1.Subject{
 				{
 					Kind:      "ServiceAccount",
-					Name:      r.Logging.QualifiedName(defaultServiceAccountName),
+					Name:      r.getServiceAccount(),
 					Namespace: r.Logging.Spec.ControlNamespace,
 				},
 			},

pkg/resources/fluentd/psp.go

@@ -32,8 +32,11 @@ func (r *Reconciler) statefulset() (runtime.Object, k8sutil.DesiredState) {
 		spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{
 			{
 				ObjectMeta: templates.FluentdObjectMeta(
-					r.Logging.QualifiedName(bufferVolumeName), util.MergeLabels(r.Logging.Labels, r.getFluentdLabels()), r.Logging),
-				Spec: r.Logging.Spec.FluentdSpec.FluentdPvcSpec,
+					r.Logging.Spec.FluentdSpec.BufferStorageVolume.PersistentVolumeClaim.PersistentVolumeSource.ClaimName,
+					util.MergeLabels(r.Logging.Labels, r.getFluentdLabels()),
+					r.Logging,
+				),
+				Spec: r.Logging.Spec.FluentdSpec.BufferStorageVolume.PersistentVolumeClaim.PersistentVolumeClaimSpec,
 				Status: corev1.PersistentVolumeClaimStatus{
 					Phase: corev1.ClaimPending,
 				},
@@ -48,6 +51,23 @@ func (r *Reconciler) statefulset() (runtime.Object, k8sutil.DesiredState) {
 }
 
 func (r *Reconciler) statefulsetSpec() *appsv1.StatefulSetSpec {
+	initContainers := make([]corev1.Container, 0)
+
+	if r.Logging.Spec.FluentdSpec.VolumeMountChmod {
+		initContainers = append(initContainers, corev1.Container{
+			Name:            "volume-mount-hack",
+			Image:           r.Logging.Spec.FluentdSpec.VolumeModImage.Repository + ":" + r.Logging.Spec.FluentdSpec.VolumeModImage.Tag,
+			ImagePullPolicy: corev1.PullPolicy(r.Logging.Spec.FluentdSpec.VolumeModImage.PullPolicy),
+			Command:         []string{"sh", "-c", "chmod -R 777 /buffers"},
+			VolumeMounts: []corev1.VolumeMount{
+				{
+					Name:      r.Logging.QualifiedName(bufferVolumeName),
+					MountPath: "/buffers",
+				},
+			},
+		})
+	}
+
 	return &appsv1.StatefulSetSpec{
 		Replicas: util.IntPointer(cast.ToInt32(r.Logging.Spec.FluentdSpec.Scaling.Replicas)),
 		Selector: &metav1.LabelSelector{
@@ -58,20 +78,7 @@ func (r *Reconciler) statefulsetSpec() *appsv1.StatefulSetSpec {
 			Spec: corev1.PodSpec{
 				Volumes:            r.generateVolume(),
 				ServiceAccountName: r.getServiceAccount(),
-				InitContainers: []corev1.Container{
-					{
-						Name:            "volume-mount-hack",
-						Image:           r.Logging.Spec.FluentdSpec.VolumeModImage.Repository + ":" + r.Logging.Spec.FluentdSpec.VolumeModImage.Tag,
-						ImagePullPolicy: corev1.PullPolicy(r.Logging.Spec.FluentdSpec.VolumeModImage.PullPolicy),
-						Command:         []string{"sh", "-c", "chmod -R 777 /buffers"},
-						VolumeMounts: []corev1.VolumeMount{
-							{
-								Name:      r.Logging.QualifiedName(bufferVolumeName),
-								MountPath: "/buffers",
-							},
-						},
-					},
-				},
+				InitContainers:     initContainers,
 				Containers: []corev1.Container{
 					*r.fluentContainer(),
 					*newConfigMapReloader(r.Logging.Spec.FluentdSpec.ConfigReloaderImage),
@@ -98,6 +105,7 @@ func (r *Reconciler) fluentContainer() *corev1.Container {
 		Resources:       r.Logging.Spec.FluentdSpec.Resources,
 		SecurityContext: &corev1.SecurityContext{
 			RunAsUser:                r.Logging.Spec.FluentdSpec.Security.SecurityContext.RunAsUser,
+			RunAsGroup:               r.Logging.Spec.FluentdSpec.Security.SecurityContext.RunAsGroup,
 			ReadOnlyRootFilesystem:   r.Logging.Spec.FluentdSpec.Security.SecurityContext.ReadOnlyRootFilesystem,
 			AllowPrivilegeEscalation: r.Logging.Spec.FluentdSpec.Security.SecurityContext.AllowPrivilegeEscalation,
 			Privileged:               r.Logging.Spec.FluentdSpec.Security.SecurityContext.Privileged,
@@ -219,26 +227,7 @@ func (r *Reconciler) generateVolume() (v []corev1.Volume) {
 			},
 		},
 	}
-	if !r.Logging.Spec.FluentdSpec.DisablePvc {
-		bufferVolume := corev1.Volume{
-			Name: r.Logging.QualifiedName(bufferVolumeName),
-			VolumeSource: corev1.VolumeSource{
-				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-					ClaimName: r.Logging.QualifiedName(bufferVolumeName),
-					ReadOnly:  false,
-				},
-			},
-		}
-		v = append(v, bufferVolume)
-	} else {
-		bufferVolume := corev1.Volume{
-			Name: r.Logging.QualifiedName(bufferVolumeName),
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{},
-			},
-		}
-		v = append(v, bufferVolume)
-	}
+	v = append(v, r.Logging.Spec.FluentdSpec.BufferStorageVolume.GetVolume(r.Logging.Name, r.Logging.QualifiedName(bufferVolumeName)))
 	if r.Logging.Spec.FluentdSpec.TLS.Enabled {
 		tlsRelatedVolume := corev1.Volume{
 			Name: "fluentd-tls",