add presave processor to deduplicate profile info

Signed-off-by: Matthias Bertschy <[email protected]>

pkg/apis/softwarecomposition/types.go

@@ -18,6 +18,7 @@ package softwarecomposition
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"strings"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -314,11 +315,37 @@ type ExecCalls struct {
 	Envs []string
 }
 
+const sep = "␟"
+
+func (e ExecCalls) String() string {
+	s := strings.Builder{}
+	s.WriteString(e.Path)
+	for _, arg := range e.Args {
+		s.WriteString(sep)
+		s.WriteString(arg)
+	}
+	for _, env := range e.Envs {
+		s.WriteString(sep)
+		s.WriteString(env)
+	}
+	return s.String()
+}
+
 type OpenCalls struct {
 	Path  string
 	Flags []string
 }
 
+func (e OpenCalls) String() string {
+	s := strings.Builder{}
+	s.WriteString(e.Path)
+	for _, arg := range e.Flags {
+		s.WriteString(sep)
+		s.WriteString(arg)
+	}
+	return s.String()
+}
+
 type ApplicationProfileStatus struct {
 }
 

pkg/apis/softwarecomposition/types_test.go

@@ -268,3 +268,80 @@ func Test_SeveritySummaryAdd(t *testing.T) {
 	}
 
 }
+
+func TestExecCalls_String(t *testing.T) {
+	tests := []struct {
+		name string
+		e    ExecCalls
+		want string
+	}{
+		{
+			name: "Empty",
+			e:    ExecCalls{},
+			want: "",
+		},
+		{
+			name: "Path only",
+			e: ExecCalls{
+				Path: "ls",
+			},
+			want: "ls",
+		},
+		{
+			name: "Path and args",
+			e: ExecCalls{
+				Path: "ls",
+				Args: []string{"-l", "-a"},
+			},
+			want: "ls␟-l␟-a",
+		},
+		{
+			name: "Path and args and env",
+			e: ExecCalls{
+				Path: "ls",
+				Args: []string{"-l", "-a"},
+				Envs: []string{"HOME=/home/user", "USER=user"},
+			},
+			want: "ls␟-l␟-a␟HOME=/home/user␟USER=user",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, tt.e.String(), "String()")
+		})
+	}
+}
+
+func TestOpenCalls_String(t *testing.T) {
+	tests := []struct {
+		name string
+		o    OpenCalls
+		want string
+	}{
+		{
+			name: "Empty",
+			o:    OpenCalls{},
+			want: "",
+		},
+		{
+			name: "Path only",
+			o: OpenCalls{
+				Path: "/etc/passwd",
+			},
+			want: "/etc/passwd",
+		},
+		{
+			name: "Path and flags",
+			o: OpenCalls{
+				Path:  "/etc/passwd",
+				Flags: []string{"O_RDONLY"},
+			},
+			want: "/etc/passwd␟O_RDONLY",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, tt.o.String(), "String()")
+		})
+	}
+}

pkg/apiserver/apiserver.go

@@ -149,6 +149,7 @@ func (c completedConfig) New() (*WardleServer, error) {
 
 	storageImpl := file.NewStorageImpl(osFs, file.DefaultStorageRoot)
 
+	applicationProfileStorageImpl := file.NewStorageImplWithCollector(osFs, file.DefaultStorageRoot, &file.ApplicationProfileProcessor{})
 	configScanStorageImpl := file.NewConfigurationScanSummaryStorage(&storageImpl)
 	vulnerabilitySummaryStorage := file.NewVulnerabilitySummaryStorage(&storageImpl)
 	generatedNetworkPolicyStorage := file.NewGeneratedNetworkPolicyStorage(&storageImpl)
@@ -168,7 +169,7 @@ func (c completedConfig) New() (*WardleServer, error) {
 	v1beta1storage["configurationscansummaries"] = sbomregistry.RESTInPeace(configurationscansummary.NewREST(Scheme, configScanStorageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["vulnerabilitysummaries"] = sbomregistry.RESTInPeace(vsumstorage.NewREST(Scheme, vulnerabilitySummaryStorage, c.GenericConfig.RESTOptionsGetter))
 
-	v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
+	v1beta1storage["applicationprofiles"] = sbomregistry.RESTInPeace(applicationprofile.NewREST(Scheme, applicationProfileStorageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["applicationprofilesummaries"] = sbomregistry.RESTInPeace(applicationprofilesummary.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
 	v1beta1storage["applicationactivities"] = sbomregistry.RESTInPeace(applicationactivity.NewREST(Scheme, storageImpl, c.GenericConfig.RESTOptionsGetter))
 

pkg/registry/file/processor.go

@@ -0,0 +1,64 @@
+package file
+
+import (
+	"fmt"
+	sets "github.com/deckarep/golang-set/v2"
+	"github.com/kubescape/storage/pkg/apis/softwarecomposition"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type Processor interface {
+	PreSave(object runtime.Object) error
+}
+
+type DefaultProcessor struct {
+}
+
+var _ Processor = (*DefaultProcessor)(nil)
+
+func (d DefaultProcessor) PreSave(_ runtime.Object) error {
+	return nil
+}
+
+type ApplicationProfileProcessor struct {
+}
+
+var _ Processor = (*ApplicationProfileProcessor)(nil)
+
+func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
+	profile, ok := object.(*softwarecomposition.ApplicationProfile)
+	if !ok {
+		return fmt.Errorf("given object is not an ApplicationProfile")
+	}
+	for i, container := range profile.Spec.Containers {
+		profile.Spec.Containers[i] = deflate(container)
+	}
+	return nil
+}
+
+func deflate(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer {
+	return softwarecomposition.ApplicationProfileContainer{
+		Name:         container.Name,
+		Capabilities: sets.NewThreadUnsafeSet(container.Capabilities...).ToSlice(),
+		Execs:        deflateStringer(container.Execs),
+		Opens:        deflateStringer(container.Opens),
+		Syscalls:     sets.NewThreadUnsafeSet(container.Syscalls...).ToSlice(),
+	}
+}
+
+type Stringer interface {
+	String() string
+}
+
+func deflateStringer[T Stringer](in []T) []T {
+	var out []T
+	set := sets.NewThreadUnsafeSet[string]()
+	for _, item := range in {
+		if set.Contains(item.String()) {
+			continue
+		}
+		set.Add(item.String())
+		out = append(out, item)
+	}
+	return out
+}

pkg/registry/file/storage.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"github.com/kubescape/go-logger"
 	"github.com/kubescape/go-logger/helpers"
+	"github.com/kubescape/storage/pkg/utils"
 	"github.com/spf13/afero"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -41,10 +42,11 @@ type objState struct {
 // hides all the storage-related operations behind it.
 type StorageImpl struct {
 	appFs           afero.Fs
-	watchDispatcher watchDispatcher
-	locks           *Mutex[string]
+	locks           *utils.Mutex[string]
+	processor       Processor
 	root            string
 	versioner       storage.Versioner
+	watchDispatcher watchDispatcher
 }
 
 // StorageQuerier wraps the storage.Interface and adds some extra methods which are used by the storage implementation.
@@ -62,10 +64,22 @@ var _ StorageQuerier = &StorageImpl{}
 func NewStorageImpl(appFs afero.Fs, root string) StorageQuerier {
 	return &StorageImpl{
 		appFs:           appFs,
+		locks:           utils.NewMapMutex[string](),
+		processor:       DefaultProcessor{},
+		root:            root,
+		versioner:       storage.APIObjectVersioner{},
 		watchDispatcher: newWatchDispatcher(),
-		locks:           NewMapMutex[string](),
+	}
+}
+
+func NewStorageImplWithCollector(appFs afero.Fs, root string, processor Processor) StorageQuerier {
+	return &StorageImpl{
+		appFs:           appFs,
+		locks:           utils.NewMapMutex[string](),
+		processor:       processor,
 		root:            root,
 		versioner:       storage.APIObjectVersioner{},
+		watchDispatcher: newWatchDispatcher(),
 	}
 }
 
@@ -106,6 +120,10 @@ func isPayloadFile(path string) bool {
 }
 
 func (s *StorageImpl) writeFiles(key string, obj runtime.Object, metaOut runtime.Object) error {
+	// call processor on object to be saved
+	if err := s.processor.PreSave(obj); err != nil {
+		return fmt.Errorf("processor.PreSave: %w", err)
+	}
 	// set resourceversion
 	if version, _ := s.versioner.ObjectResourceVersion(obj); version == 0 {
 		if err := s.versioner.UpdateObject(obj, 1); err != nil {
@@ -470,7 +488,7 @@ func (s *StorageImpl) GuaranteedUpdate(
 		if err != nil {
 			// If our data is already up-to-date, return the error
 			if origStateIsCurrent {
-				if !apierrors.IsNotFound(err) {
+				if !apierrors.IsNotFound(err) && !apierrors.IsInvalid(err) {
 					logger.L().Ctx(ctx).Error("tryUpdate func failed", helpers.Error(err), helpers.String("key", key))
 				}
 				return err
@@ -491,7 +509,7 @@ func (s *StorageImpl) GuaranteedUpdate(
 
 			// it turns out our cached data was not stale, return the error
 			if cachedRev == origState.rev {
-				if !apierrors.IsNotFound(err) {
+				if !apierrors.IsNotFound(err) && !apierrors.IsInvalid(err) {
 					logger.L().Ctx(ctx).Error("tryUpdate func failed", helpers.Error(err), helpers.String("key", key))
 				}
 				return cachedUpdateErr

pkg/registry/file/mutex.go → pkg/utils/mutex.go

@@ -1,4 +1,4 @@
-package file
+package utils
 
 import (
 	"math/rand"

pkg/registry/file/mutex_test.go → pkg/utils/mutex_test.go

@@ -1,4 +1,4 @@
-package file
+package utils
 
 import (
 	"fmt"