Merge pull request kubearmor#1335 from rksharma95/feat-multienforcer-…

…controller

add multienforcer controller

.github/workflows/ci-test-ginkgo.yml

@@ -42,6 +42,14 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: 'KubeArmor/go.mod'
+
+      - name: Check what paths were updated
+        uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            controller:
+              - 'pkg/KubeArmorController/**'
 
       - name: Install the latest LLVM toolchain
         run: ./.github/workflows/install-llvm.sh
@@ -60,6 +68,10 @@ jobs:
         working-directory: pkg/KubeArmorOperator
         run: |
           make docker-build
+      
+      - name: Build KubeArmorController
+        if: steps.filter.outputs.controller == 'true'
+        run: make -C pkg/KubeArmorController/ docker-build TAG=latest
 
       - name: deploy pre existing pod 
         run: |
@@ -69,11 +81,15 @@ jobs:
 
       - name: Run KubeArmor
         run: |
-          if [ ${{ matrix.runtime }} == "containerd" ]; then
+          if [[ ${{ matrix.runtime }} == "containerd" ]]; then
             docker save kubearmor/kubearmor-init:latest | sudo k3s ctr images import -
             docker save kubearmor/kubearmor:latest | sudo k3s ctr images import -
             docker save kubearmor/kubearmor-operator:latest | sudo k3s ctr images import -
             docker save kubearmor/kubearmor-snitch:latest | sudo k3s ctr images import -
+            
+            if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then 
+              docker save kubearmor/kubearmor-controller:latest | sudo k3s ctr images import -
+            fi
           else
             if [ ${{ matrix.runtime }} == "crio" ]; then
               docker save kubearmor/kubearmor-init:latest | sudo podman load
@@ -84,12 +100,25 @@ jobs:
               sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-operator:latest
               docker save kubearmor/kubearmor-snitch:latest | sudo podman load
               sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-snitch:latest
+              if [ ${{ steps.filter.outputs.controller }} == 'true' ]; then
+                docker save kubearmor/kubearmor-controller:latest | sudo podman load
+                sudo podman tag localhost/latest:latest docker.io/kubearmor/kubearmor-controller:latest
+              fi
             fi
           fi
-          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
+          docker system prune -a -f 
+          docker buildx prune -a -f
+          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest 
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A
-          kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml
+          if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml --dry-run=client -o json | \
+            jq '.spec.kubearmorControllerImage.imagePullPolicy = "Never"' | \
+            kubectl apply -f -
+          else 
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-test.yaml
+          fi
+
           kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test
           kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor
           kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor

.github/workflows/ci-test-ubi-image.yml

@@ -45,6 +45,14 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: 'KubeArmor/go.mod'
+
+      - name: Check what paths were updated
+        uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            controller:
+              - 'pkg/KubeArmorController/**'
  
       - name: Install the latest LLVM toolchain
         run: ./.github/workflows/install-llvm.sh
@@ -63,17 +71,34 @@ jobs:
         working-directory: pkg/KubeArmorOperator
         run: |
           make docker-build
+      
+      - name: Build KubeArmorController
+        if: steps.filter.outputs.controller == 'true'
+        run: make -C pkg/KubeArmorController/ docker-build TAG=latest
 
       - name: Run KubeArmor
         run: |
           docker save kubearmor/kubearmor-init:latest | sudo podman load
           docker save kubearmor/kubearmor-ubi:latest | sudo podman load
           docker save kubearmor/kubearmor-operator:latest | sudo podman load
           docker save kubearmor/kubearmor-snitch:latest | sudo podman load
+          
+          if [ ${{ steps.filter.outputs.controller }} == 'true' ]; then 
+            docker save kubearmor/kubearmor-controller:latest | sudo podman load
+          fi          
+          
           helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest
           kubectl get pods -A
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
-          kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-ubi-test.yaml
+
+          if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-ubi-test.yaml --dry-run=client -o json | \
+            jq '.spec.kubearmorControllerImage.imagePullPolicy = "Never"' | \
+            kubectl apply -f -
+          else 
+            kubectl apply -f pkg/KubeArmorOperator/config/samples/kubearmor-ubi-test.yaml
+          fi
+
           kubectl wait -n kubearmor --timeout=5m --for=jsonpath='{.status.phase}'=Running kubearmorconfigs/kubearmorconfig-test
           kubectl wait --timeout=7m --for=condition=ready pod -l kubearmor-app,kubearmor-app!=kubearmor-snitch,kubearmor-app!=kubearmor-controller -n kubearmor
           kubectl wait --timeout=1m --for=condition=ready pod -l kubearmor-app=kubearmor-controller -n kubearmor

KubeArmor/Makefile

@@ -51,7 +51,7 @@ run: build
 	cd $(CURDIR); sudo rm -f /tmp/kubearmor.log
 	cd $(CURDIR)/BPF; make clean
 	cd $(CURDIR)/BPF; make
-	cd $(CURDIR); DEBUG=true sudo -E ./kubearmor -logPath=/tmp/kubearmor.log -enableKubeArmorPolicy -enableKubeArmorHostPolicy -hostVisibility=process,file,network,capabilities -defaultFilePosture block -defaultCapabilitiesPosture block -defaultNetworkPosture block -hostDefaultFilePosture block -hostDefaultCapabilitiesPosture block -hostDefaultNetworkPosture block
+	cd $(CURDIR); DEBUG=true sudo -E ./kubearmor -logPath=/tmp/kubearmor.log -enableKubeArmorPolicy -enableKubeArmorHostPolicy -hostVisibility=process,file,network,capabilities -defaultFilePosture block -defaultCapabilitiesPosture block -defaultNetworkPosture block -hostDefaultFilePosture block -hostDefaultCapabilitiesPosture block -hostDefaultNetworkPosture block -annotateResources=true
 
 .PHONY: run-container
 run-container: build

KubeArmor/config/config.go

@@ -56,9 +56,10 @@ type KubearmorConfig struct {
 
 	StateAgent bool // enable KubeArmor state agent
 
-	AlertThrottling bool // Enable/Disable Alert Throttling
-	MaxAlertPerSec  int  // Maximum alerts allowed per second
-	ThrottleSec     int  // Number of seconds for which subsequent alerts will be dropped
+	AlertThrottling   bool // Enable/Disable Alert Throttling
+	MaxAlertPerSec    int  // Maximum alerts allowed per second
+	ThrottleSec       int  // Number of seconds for which subsequent alerts will be dropped
+	AnnotateResources bool // enable annotations by kubearmor if kubearmor-controller is not present
 }
 
 // GlobalCfg Global configuration for Kubearmor
@@ -103,6 +104,7 @@ const (
 	ConfigAlertThrottling                string = "alertThrottling"
 	ConfigMaxAlertPerSec                 string = "maxAlertPerSec"
 	ConfigThrottleSec                    string = "throttleSec"
+	ConfigAnnotateResources              string = "annotateResources"
 )
 
 func readCmdLineParams() {
@@ -157,6 +159,8 @@ func readCmdLineParams() {
 
 	throttleSec := flag.Int(ConfigThrottleSec, 30, "Time period for which subsequent alerts will be dropped (in sec)")
 
+	annotateResources := flag.Bool(ConfigAnnotateResources, false, "for kubearmor deployment without kubearmor-controller")
+
 	flags := []string{}
 	flag.VisitAll(func(f *flag.Flag) {
 		kv := fmt.Sprintf("%s:%v", f.Name, f.Value)
@@ -212,8 +216,12 @@ func readCmdLineParams() {
 	viper.SetDefault(ConfigStateAgent, *stateAgent)
 
 	viper.SetDefault(ConfigAlertThrottling, *alertThrottling)
+
 	viper.SetDefault(ConfigMaxAlertPerSec, *maxAlertPerSec)
+
 	viper.SetDefault(ConfigThrottleSec, *throttleSec)
+
+	viper.SetDefault(ConfigAnnotateResources, *annotateResources)
 }
 
 // LoadConfig Load configuration
@@ -312,6 +320,7 @@ func LoadConfig() error {
 	GlobalCfg.AlertThrottling = viper.GetBool(ConfigAlertThrottling)
 	GlobalCfg.MaxAlertPerSec = viper.GetInt(ConfigMaxAlertPerSec)
 	GlobalCfg.ThrottleSec = viper.GetInt(ConfigThrottleSec)
+	GlobalCfg.AnnotateResources = viper.GetBool(ConfigAnnotateResources)
 
 	kg.Printf("Final Configuration [%+v]", GlobalCfg)
 

KubeArmor/core/kubeUpdate.go

@@ -839,7 +839,9 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 						dm.RuntimeEnforcer.UpdateAppArmorProfiles(pod.Metadata["podName"], "ADDED", appArmorAnnotations, pod.PrivilegedAppArmorProfiles)
 
 						if updateAppArmor && pod.Annotations["kubearmor-policy"] == "enabled" && dm.OwnerInfo[pod.Metadata["podName"]].Ref != "Pod" {
-							if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" {
+
+							// patch deployments only when kubearmor-controller is not present
+							if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" && cfg.GlobalCfg.AnnotateResources {
 								deploymentName := dm.OwnerInfo[pod.Metadata["podName"]].Name
 								// patch the deployment with apparmor annotations
 								if err := K8s.PatchResourceWithAppArmorAnnotations(pod.Metadata["namespaceName"], deploymentName, appArmorAnnotations, dm.OwnerInfo[pod.Metadata["podName"]].Ref); err != nil {
@@ -860,7 +862,9 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 								}
 
 								if updateAppArmor && prevPolicyEnabled != "enabled" && pod.Annotations["kubearmor-policy"] == "enabled" && dm.OwnerInfo[pod.Metadata["podName"]].Ref != "Pod" {
-									if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" {
+
+									// patch deployments only when kubearmor-controller is not present
+									if dm.OwnerInfo[pod.Metadata["podName"]].Name != "" && cfg.GlobalCfg.AnnotateResources {
 										deploymentName := dm.OwnerInfo[pod.Metadata["podName"]].Name
 										// patch the deployment with apparmor annotations
 										if err := K8s.PatchResourceWithAppArmorAnnotations(pod.Metadata["namespaceName"], deploymentName, appArmorAnnotations, dm.OwnerInfo[pod.Metadata["podName"]].Ref); err != nil {
@@ -885,7 +889,6 @@ func (dm *KubeArmorDaemon) WatchK8sPods() {
 
 				if event.Type == "ADDED" {
 					new := true
-
 					for _, k8spod := range dm.K8sPods {
 						if k8spod.Metadata["namespaceName"] == pod.Metadata["namespaceName"] && k8spod.Metadata["podName"] == pod.Metadata["podName"] {
 							new = false

KubeArmor/go.mod

@@ -44,7 +44,7 @@ require (
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
 	k8s.io/client-go v0.29.0
-	k8s.io/cri-api v0.29.0
+	k8s.io/cri-api v0.29.7
 	k8s.io/klog/v2 v2.120.0
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
 	sigs.k8s.io/controller-runtime v0.15.3
@@ -65,7 +65,7 @@ require (
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.20.2 // indirect
 	github.com/go-openapi/jsonreference v0.20.4 // indirect

KubeArmor/go.sum

@@ -58,8 +58,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
-github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
@@ -376,8 +376,8 @@ k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
 k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
 k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s=
 k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
-k8s.io/cri-api v0.29.0 h1:atenAqOltRsFqcCQlFFpDnl/R4aGfOELoNLTDJfd7t8=
-k8s.io/cri-api v0.29.0/go.mod h1:Rls2JoVwfC7kW3tndm7267kriuRukQ02qfht0PCRuIc=
+k8s.io/cri-api v0.29.7 h1:5X1Fid6oxYsP9/W1NtX0RYUefM2UNwaqfew8z7Pbf/M=
+k8s.io/cri-api v0.29.7/go.mod h1:A6pdbjzML2xi9B0Clqn5qt1HJ3Ik12x2j+jv/TkqjRE=
 k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8=
 k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU=

deployments/get/objects.go

@@ -507,16 +507,6 @@ var KubeArmorControllerCertVolume = corev1.Volume{
 	},
 }
 
-var KubeArmorControllerHostPathVolume = corev1.Volume{
-	Name: "sys-path",
-	VolumeSource: corev1.VolumeSource{
-		HostPath: &corev1.HostPathVolumeSource{
-			Path: "/sys/kernel/security",
-			Type: &hostPathDirectory,
-		},
-	},
-}
-
 var KubeArmorControllerAllowPrivilegeEscalation = false
 
 // GetKubeArmorControllerDeployment Function
@@ -549,7 +539,6 @@ func GetKubeArmorControllerDeployment(namespace string) *appsv1.Deployment {
 					ServiceAccountName: KubeArmorControllerServiceAccountName,
 					Volumes: []corev1.Volume{
 						KubeArmorControllerCertVolume,
-						KubeArmorControllerHostPathVolume,
 					},
 					Containers: []corev1.Container{
 						{
@@ -600,11 +589,6 @@ func GetKubeArmorControllerDeployment(namespace string) *appsv1.Deployment {
 									ReadOnly:  true,
 									MountPath: "/tmp/k8s-webhook-server/serving-certs",
 								},
-								{
-									Name:      KubeArmorControllerHostPathVolume.Name,
-									ReadOnly:  true,
-									MountPath: "/sys/kernel/security",
-								},
 							},
 							SecurityContext: &corev1.SecurityContext{
 								AllowPrivilegeEscalation: &KubeArmorControllerAllowPrivilegeEscalation,
@@ -674,6 +658,11 @@ func GetKubeArmorControllerClusterRole() *rbacv1.ClusterRole {
 				Resources: []string{"pods"},
 				Verbs:     []string{"create", "delete", "get", "patch", "list", "watch", "update"},
 			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"nodes"},
+				Verbs:     []string{"get", "list", "watch"},
+			},
 			{
 				APIGroups: []string{"security.kubearmor.com"},
 				Resources: []string{"kubearmorpolicies", "kubearmorclusterpolicies", "kubearmorhostpolicies"},

deployments/helm/KubeArmor/templates/RBAC/roles.yaml

@@ -88,6 +88,14 @@ rules:
   - list
   - watch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch   
 - apiGroups:
   - security.kubearmor.com
   resources:

deployments/helm/KubeArmorOperator/templates/clusterrole-rbac.yaml

@@ -176,6 +176,14 @@ rules:
   - list
   - watch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs: 
+  - get 
+  - list 
+  - watch
 - apiGroups:
   - security.kubearmor.com
   resources:

pkg/KubeArmorController/Dockerfile

@@ -17,6 +17,9 @@ COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
 COPY handlers/ handlers/
+COPY informer/ informer/
+COPY types/ types/
+COPY common/ common/
 
 # Build
 RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o manager main.go