Merge pull request #148 from kubescape/feature/generic_path_analyzer

Added dynamic open path detector

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/go-logr/zapr v1.2.4
 	github.com/gogo/protobuf v1.3.2
 	github.com/goradd/maps v0.1.5
+	github.com/kinbiko/jsonassert v1.1.1
 	github.com/kubescape/go-logger v0.0.22
 	github.com/kubescape/k8s-interface v0.0.162
 	github.com/olvrng/ujson v1.1.0

go.sum

@@ -1033,6 +1033,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kinbiko/jsonassert v1.1.1 h1:DB12divY+YB+cVpHULLuKePSi6+ui4M/shHSzJISkSE=
+github.com/kinbiko/jsonassert v1.1.1/go.mod h1:NO4lzrogohtIdNUNzx8sdzB55M4R4Q1bsrWVdqQ7C+A=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=

pkg/apis/softwarecomposition/networkpolicy/v2/networkpolicy_test.go

@@ -1387,9 +1387,7 @@ func TestRemoveLabels(t *testing.T) {
 
 	removeLabels(labels)
 
-	if !reflect.DeepEqual(labels, expected) {
-		t.Errorf("removeLabels() = %v, want %v", labels, expected)
-	}
+	assert.Equal(t, expected, labels)
 }
 
 func TestMergeIngressRulesByPorts(t *testing.T) {

pkg/registry/file/applicationprofile_processor.go

@@ -13,6 +13,11 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+const (
+	OpenDynamicThreshold     = 50
+	EndpointDynamicThreshold = 100
+)
+
 type ApplicationProfileProcessor struct {
 }
 
@@ -53,16 +58,26 @@ func (a ApplicationProfileProcessor) PreSave(object runtime.Object) error {
 }
 
 func deflateApplicationProfileContainer(container softwarecomposition.ApplicationProfileContainer) softwarecomposition.ApplicationProfileContainer {
-	endpoints, err := dynamicpathdetector.AnalyzeEndpoints(&container.Endpoints, dynamicpathdetector.NewPathAnalyzer(100))
+	opens, err := dynamicpathdetector.AnalyzeOpens(container.Opens, dynamicpathdetector.NewPathAnalyzer(OpenDynamicThreshold))
+	if err != nil {
+		logger.L().Warning("failed to analyze opens", loggerhelpers.Error(err))
+		opens = DeflateStringer(container.Opens)
+	}
+
+	if opens == nil {
+		opens = []softwarecomposition.OpenCalls{}
+	}
+
+	endpoints, err := dynamicpathdetector.AnalyzeEndpoints(&container.Endpoints, dynamicpathdetector.NewPathAnalyzer(EndpointDynamicThreshold))
 	if err != nil {
 		logger.L().Warning("failed to analyze endpoints", loggerhelpers.Error(err))
 		endpoints = container.Endpoints
 	}
 	return softwarecomposition.ApplicationProfileContainer{
 		Name:           container.Name,
 		Capabilities:   mapset.Sorted(mapset.NewThreadUnsafeSet(container.Capabilities...)),
-		Execs:          deflateStringer(container.Execs),
-		Opens:          deflateStringer(container.Opens),
+		Execs:          DeflateStringer(container.Execs),
+		Opens:          opens,
 		Syscalls:       mapset.Sorted(mapset.NewThreadUnsafeSet(container.Syscalls...)),
 		SeccompProfile: container.SeccompProfile,
 		Endpoints:      endpoints,

pkg/registry/file/dynamicpathdetector/analyze_endpoints.go

@@ -48,7 +48,7 @@ func ProcessEndpoint(endpoint *types.HTTPEndpoint, analyzer *PathAnalyzer, newEn
 		// Check if this dynamic exists
 		for i, e := range newEndpoints {
 			if e.Endpoint == url {
-				newEndpoints[i].Methods = mergeMethods(e.Methods, endpoint.Methods)
+				newEndpoints[i].Methods = MergeStrings(e.Methods, endpoint.Methods)
 				mergeHeaders(e, endpoint)
 				return nil, nil
 			}
@@ -98,7 +98,7 @@ func MergeDuplicateEndpoints(endpoints []*types.HTTPEndpoint) ([]*types.HTTPEndp
 		key := getEndpointKey(endpoint)
 
 		if existing, found := seen[key]; found {
-			existing.Methods = mergeMethods(existing.Methods, endpoint.Methods)
+			existing.Methods = MergeStrings(existing.Methods, endpoint.Methods)
 			mergeHeaders(existing, endpoint)
 		} else {
 			seen[key] = endpoint
@@ -142,21 +142,6 @@ func mergeHeaders(existing, new *types.HTTPEndpoint) {
 	existing.Headers = rawJSON
 }
 
-func mergeMethods(existing, new []string) []string {
-	methodSet := make(map[string]bool)
-	for _, m := range existing {
-		methodSet[m] = true
-	}
-	for _, m := range new {
-		if !methodSet[m] {
-			existing = append(existing, m)
-			methodSet[m] = true
-		}
-	}
-
-	return existing
-}
-
 func convertPointerToValueSlice(m []*types.HTTPEndpoint) []types.HTTPEndpoint {
 	result := make([]types.HTTPEndpoint, 0, len(m))
 	for _, v := range m {

pkg/registry/file/dynamicpathdetector/analyze_opens.go

@@ -0,0 +1,49 @@
+package dynamicpathdetector
+
+import (
+	"fmt"
+	"slices"
+
+	mapset "github.com/deckarep/golang-set/v2"
+	types "github.com/kubescape/storage/pkg/apis/softwarecomposition"
+)
+
+func AnalyzeOpens(opens []types.OpenCalls, analyzer *PathAnalyzer) ([]types.OpenCalls, error) {
+	var dynamicOpens []types.OpenCalls
+	for _, open := range opens {
+		_, _ = AnalyzeOpen(open.Path, analyzer)
+	}
+
+	for i := range opens {
+		result, err := AnalyzeOpen(opens[i].Path, analyzer)
+		if err != nil {
+			continue
+		}
+
+		if result != opens[i].Path {
+			if existing, err := getIfExists(result, dynamicOpens); err == nil {
+				existing.Flags = mapset.Sorted(mapset.NewThreadUnsafeSet(slices.Concat(existing.Flags, opens[i].Flags)...))
+			} else {
+				dynamicOpen := types.OpenCalls{Path: result, Flags: opens[i].Flags}
+				dynamicOpens = append(dynamicOpens, dynamicOpen)
+			}
+		} else {
+			dynamicOpens = append(dynamicOpens, opens[i])
+		}
+	}
+
+	return dynamicOpens, nil
+}
+
+func AnalyzeOpen(path string, analyzer *PathAnalyzer) (string, error) {
+	return analyzer.AnalyzePath(path, "opens")
+}
+
+func getIfExists(path string, dynamicOpens []types.OpenCalls) (*types.OpenCalls, error) {
+	for i := range dynamicOpens {
+		if dynamicOpens[i].Path == path {
+			return &dynamicOpens[i], nil
+		}
+	}
+	return nil, fmt.Errorf("not found")
+}

pkg/registry/file/dynamicpathdetector/analyzer.go

@@ -1,7 +1,7 @@
 package dynamicpathdetector
 
 import (
-	pathUtils "path"
+	"path"
 	"strings"
 )
 
@@ -11,8 +11,9 @@ func NewPathAnalyzer(threshold int) *PathAnalyzer {
 		threshold: threshold,
 	}
 }
-func (ua *PathAnalyzer) AnalyzePath(path, identifier string) (string, error) {
-	path = pathUtils.Clean(path)
+
+func (ua *PathAnalyzer) AnalyzePath(p, identifier string) (string, error) {
+	p = path.Clean(p)
 	node, exists := ua.RootNodes[identifier]
 	if !exists {
 		node = &SegmentNode{
@@ -22,43 +23,48 @@ func (ua *PathAnalyzer) AnalyzePath(path, identifier string) (string, error) {
 		}
 		ua.RootNodes[identifier] = node
 	}
-
-	segments := strings.Split(strings.Trim(path, "/"), "/")
-
-	return ua.processSegments(node, segments), nil
+	return ua.processSegments(node, p), nil
 }
 
-func (ua *PathAnalyzer) processSegments(node *SegmentNode, segments []string) string {
-	resultPath := []string{}
+func (ua *PathAnalyzer) processSegments(node *SegmentNode, p string) string {
+	var result strings.Builder
 	currentNode := node
-	for _, segment := range segments {
+	start := 0
+	for i := range p {
+		if p[i] == '/' {
+			segment := p[start:i]
+			currentNode = ua.processSegment(currentNode, segment)
+			ua.updateNodeStats(currentNode)
+			result.WriteString(currentNode.SegmentName)
+			result.WriteByte('/')
+			start = i + 1
+		}
+	}
+	// Process the last segment
+	if start < len(p) {
+		segment := p[start:]
 		currentNode = ua.processSegment(currentNode, segment)
 		ua.updateNodeStats(currentNode)
-		resultPath = append(resultPath, currentNode.SegmentName)
+		result.WriteString(currentNode.SegmentName)
 	}
-	return "/" + strings.Join(resultPath, "/")
-
+	return result.String()
 }
 
 func (ua *PathAnalyzer) processSegment(node *SegmentNode, segment string) *SegmentNode {
-
-	switch {
-	case segment == dynamicIdentifier:
+	if segment == DynamicIdentifier {
 		return ua.handleDynamicSegment(node)
-	case KeyInMap(node.Children, segment) || node.IsNextDynamic():
-		child, exists := node.Children[segment]
+	} else if child, exists := node.Children[segment]; exists || node.IsNextDynamic() {
 		return ua.handleExistingSegment(node, child, exists)
-	default:
+	} else {
 		return ua.handleNewSegment(node, segment)
-
 	}
 }
 
 func (ua *PathAnalyzer) handleExistingSegment(node *SegmentNode, child *SegmentNode, exists bool) *SegmentNode {
 	if exists {
 		return child
 	} else {
-		return node.Children[dynamicIdentifier]
+		return node.Children[DynamicIdentifier]
 	}
 }
 
@@ -74,7 +80,7 @@ func (ua *PathAnalyzer) handleNewSegment(node *SegmentNode, segment string) *Seg
 }
 
 func (ua *PathAnalyzer) handleDynamicSegment(node *SegmentNode) *SegmentNode {
-	if dynamicChild, exists := node.Children[dynamicIdentifier]; exists {
+	if dynamicChild, exists := node.Children[DynamicIdentifier]; exists {
 		return dynamicChild
 	} else {
 		return ua.createDynamicNode(node)
@@ -83,7 +89,7 @@ func (ua *PathAnalyzer) handleDynamicSegment(node *SegmentNode) *SegmentNode {
 
 func (ua *PathAnalyzer) createDynamicNode(node *SegmentNode) *SegmentNode {
 	dynamicNode := &SegmentNode{
-		SegmentName: dynamicIdentifier,
+		SegmentName: DynamicIdentifier,
 		Count:       0,
 		Children:    make(map[string]*SegmentNode),
 	}
@@ -95,17 +101,16 @@ func (ua *PathAnalyzer) createDynamicNode(node *SegmentNode) *SegmentNode {
 
 	// Replace all children with the new dynamic node
 	node.Children = map[string]*SegmentNode{
-		dynamicIdentifier: dynamicNode,
+		DynamicIdentifier: dynamicNode,
 	}
 
 	return dynamicNode
 }
 
 func (ua *PathAnalyzer) updateNodeStats(node *SegmentNode) {
 	if node.Count > ua.threshold && !node.IsNextDynamic() {
-
 		dynamicChild := &SegmentNode{
-			SegmentName: dynamicIdentifier,
+			SegmentName: DynamicIdentifier,
 			Count:       0,
 			Children:    make(map[string]*SegmentNode),
 		}
@@ -116,23 +121,18 @@ func (ua *PathAnalyzer) updateNodeStats(node *SegmentNode) {
 		}
 
 		node.Children = map[string]*SegmentNode{
-			dynamicIdentifier: dynamicChild,
+			DynamicIdentifier: dynamicChild,
 		}
 	}
 }
 
 func shallowChildrenCopy(src, dst *SegmentNode) {
 	for segmentName := range src.Children {
-		if !KeyInMap(dst.Children, segmentName) {
+		if _, ok := dst.Children[segmentName]; !ok {
 			dst.Children[segmentName] = src.Children[segmentName]
 		} else {
 			dst.Children[segmentName].Count += src.Children[segmentName].Count
 			shallowChildrenCopy(src.Children[segmentName], dst.Children[segmentName])
 		}
 	}
 }
-
-func KeyInMap[T any](TestMap map[string]T, key string) bool {
-	_, ok := TestMap[key]
-	return ok
-}