Merge branch http-object-type into http-object-type-mb

pkg/apis/softwarecomposition/types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package softwarecomposition
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/containers/common/pkg/seccomp"
@@ -633,6 +634,56 @@ func (e *HTTPEndpoint) Equal(other *HTTPEndpoint) bool {
 	return e.Endpoint == other.Endpoint && e.Direction == other.Direction && e.Internal == other.Internal
 }
 
+func (e HTTPEndpoint) String() string {
+	const sep = "␟"
+	var s strings.Builder
+
+	// Append Endpoint
+	s.WriteString(e.Endpoint)
+
+	// Append Methods
+	if len(e.Methods) > 0 {
+		if s.Len() > 0 {
+			s.WriteString(sep)
+		}
+		s.WriteString(strings.Join(e.Methods, ","))
+	}
+
+	// Append Internal status
+	if e.Internal {
+		if s.Len() > 0 {
+			s.WriteString(sep)
+		}
+		s.WriteString("Internal")
+	}
+
+	// Append Direction
+	if e.Direction != "" {
+		if s.Len() > 0 {
+			s.WriteString(sep)
+		}
+		// Capitalize the first letter of the direction
+		s.WriteString(strings.Title(string(e.Direction)))
+	}
+
+	// Append Headers
+	if len(e.Headers) > 0 {
+		// Define the order of headers
+		orderedHeaders := []string{"Content-Type", "Authorization"}
+
+		for _, k := range orderedHeaders {
+			if values, ok := e.Headers[k]; ok {
+				if s.Len() > 0 {
+					s.WriteString(sep)
+				}
+				s.WriteString(fmt.Sprintf("%s: %s", k, strings.Join(values, ",")))
+			}
+		}
+	}
+
+	return s.String()
+}
+
 type SpecBase struct {
 	Disabled bool
 }

pkg/apis/softwarecomposition/types_test.go

@@ -3,6 +3,7 @@ package softwarecomposition
 import (
 	"testing"
 
+	"github.com/kubescape/storage/pkg/apis/softwarecomposition/consts"
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -345,3 +346,44 @@ func TestOpenCalls_String(t *testing.T) {
 		})
 	}
 }
+
+func TestHTTPEndpoint_String(t *testing.T) {
+	tests := []struct {
+		name string
+		e    HTTPEndpoint
+		want string
+	}{
+		{
+			name: "Empty",
+			e:    HTTPEndpoint{},
+			want: "",
+		},
+		{
+			name: "Endpoint and Methods only",
+			e: HTTPEndpoint{
+				Endpoint: "/api/v1/users",
+				Methods:  []string{"GET", "POST"},
+			},
+			want: "/api/v1/users␟GET,POST",
+		},
+		{
+			name: "Full HTTPEndpoint",
+			e: HTTPEndpoint{
+				Endpoint:  "/api/v1/users",
+				Methods:   []string{"GET", "POST"},
+				Internal:  consts.True,
+				Direction: consts.Inbound,
+				Headers: map[string][]string{
+					"Content-Type":  {"application/json"},
+					"Authorization": {"Bearer token123", "ApiKey abcdef"},
+				},
+			},
+			want: "/api/v1/users␟GET,POST␟Internal␟Inbound␟Content-Type: application/json␟Authorization: Bearer token123,ApiKey abcdef",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, tt.e.String(), "String()")
+		})
+	}
+}

pkg/registry/file/applicationprofile_processor_test.go

@@ -127,7 +127,7 @@ func TestApplicationProfileProcessor_PreSave(t *testing.T) {
 							Syscalls: []string{},
 							Endpoints: []softwarecomposition.HTTPEndpoint{
 								{
-									Endpoint:  "http://localhost:8080",
+									Endpoint:  "localhost/",
 									Methods:   []string{"GET"},
 									Internal:  false,
 									Direction: consts.Inbound,

pkg/registry/file/dynamicpathdetector/analyze_endpoints.go

@@ -72,6 +72,9 @@ func AnalyzeURL(urlString string, analyzer *PathAnalyzer) (string, error) {
 	hostname := parsedURL.Hostname()
 
 	path, _ := analyzer.AnalyzePath(parsedURL.Path, hostname)
+	if path == "/." {
+		path = "/"
+	}
 	return hostname + path, nil
 }
 

pkg/registry/file/dynamicpathdetector/analyzer.go

@@ -1,24 +1,25 @@
 package dynamicpathdetector
 
 import (
+	pathUtils "path"
 	"strings"
 )
 
 func NewPathAnalyzer() *PathAnalyzer {
 	return &PathAnalyzer{
-		rootNodes: make(map[string]*SegmentNode),
+		RootNodes: make(map[string]*SegmentNode),
 	}
 }
 func (ua *PathAnalyzer) AnalyzePath(path, identifier string) (string, error) {
-
-	node, exists := ua.rootNodes[identifier]
+	path = pathUtils.Clean(path)
+	node, exists := ua.RootNodes[identifier]
 	if !exists {
 		node = &SegmentNode{
 			SegmentName: identifier,
 			Count:       0,
 			Children:    make(map[string]*SegmentNode),
 		}
-		ua.rootNodes[identifier] = node
+		ua.RootNodes[identifier] = node
 	}
 
 	segments := strings.Split(strings.Trim(path, "/"), "/")
@@ -88,7 +89,7 @@ func (ua *PathAnalyzer) createDynamicNode(node *SegmentNode) *SegmentNode {
 
 	// Copy all existing children to the new dynamic node
 	for _, child := range node.Children {
-		shallowChildrensCopy(child, dynamicNode)
+		shallowChildrenCopy(child, dynamicNode)
 	}
 
 	// Replace all children with the new dynamic node
@@ -100,7 +101,7 @@ func (ua *PathAnalyzer) createDynamicNode(node *SegmentNode) *SegmentNode {
 }
 
 func (ua *PathAnalyzer) updateNodeStats(node *SegmentNode) {
-	if node.Count > theshold && !node.IsNextDynamic() {
+	if node.Count > threshold && !node.IsNextDynamic() {
 
 		dynamicChild := &SegmentNode{
 			SegmentName: dynamicIdentifier,
@@ -110,7 +111,7 @@ func (ua *PathAnalyzer) updateNodeStats(node *SegmentNode) {
 
 		// Copy all descendants
 		for _, child := range node.Children {
-			shallowChildrensCopy(child, dynamicChild)
+			shallowChildrenCopy(child, dynamicChild)
 		}
 
 		node.Children = map[string]*SegmentNode{
@@ -119,10 +120,13 @@ func (ua *PathAnalyzer) updateNodeStats(node *SegmentNode) {
 	}
 }
 
-func shallowChildrensCopy(src, dst *SegmentNode) {
+func shallowChildrenCopy(src, dst *SegmentNode) {
 	for segmentName := range src.Children {
 		if !KeyInMap(dst.Children, segmentName) {
 			dst.Children[segmentName] = src.Children[segmentName]
+		} else {
+			dst.Children[segmentName].Count += src.Children[segmentName].Count
+			shallowChildrenCopy(src.Children[segmentName], dst.Children[segmentName])
 		}
 	}
 }

pkg/registry/file/dynamicpathdetector/tests/benchamark_test.go

@@ -0,0 +1,116 @@
+package dynamicpathdetectortests
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"github.com/kubescape/storage/pkg/registry/file/dynamicpathdetector"
+)
+
+func BenchmarkAnalyzePath(b *testing.B) {
+	analyzer := dynamicpathdetector.NewPathAnalyzer()
+	paths := generateMixedPaths(10000, 0) // 0 means use default mixed lengths
+
+	identifier := "test"
+
+	// Ensure we analyze at least 10,000 paths
+	minIterations := 10000
+	if b.N < minIterations {
+		b.N = minIterations
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		path := paths[i%len(paths)]
+		_, err := analyzer.AnalyzePath(path, identifier)
+		if err != nil {
+			b.Fatalf("Error analyzing path: %v", err)
+		}
+	}
+}
+
+func BenchmarkAnalyzePathWithDifferentLengths(b *testing.B) {
+	pathLengths := []int{1, 3, 5, 10, 20, 50, 100}
+
+	for _, length := range pathLengths {
+		b.Run(fmt.Sprintf("PathLength-%d", length), func(b *testing.B) {
+			analyzer := dynamicpathdetector.NewPathAnalyzer()
+			paths := generateMixedPaths(10000, length)
+			identifier := "test"
+
+			// Ensure we analyze at least 10,000 paths
+			minIterations := 10000
+			if b.N < minIterations {
+				b.N = minIterations
+			}
+
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				path := paths[i%len(paths)]
+				_, err := analyzer.AnalyzePath(path, identifier)
+				if err != nil {
+					b.Fatalf("Error analyzing path: %v", err)
+				}
+			}
+
+		})
+	}
+}
+
+func generateMixedPaths(count int, fixedLength int) []string {
+	paths := make([]string, count)
+	staticSegments := []string{"users", "profile", "settings", "api", "v1", "posts", "organizations", "departments", "employees", "projects", "tasks", "categories", "subcategories", "items", "articles"}
+
+	for i := 0; i < count; i++ {
+		if fixedLength > 0 {
+			segments := make([]string, fixedLength)
+			for j := 0; j < fixedLength; j++ {
+				if rand.Float32() < 0.2 { // 20% chance of dynamic segment
+					prefix := staticSegments[rand.Intn(len(staticSegments))]
+					// Generate a value > 100 to ensure it's considered dynamic
+					dynamicValue := rand.Intn(10000) + 101
+					segments[j] = fmt.Sprintf("%s_%d", prefix, dynamicValue)
+				} else { // 80% chance of static segment
+					segments[j] = staticSegments[rand.Intn(len(staticSegments))]
+				}
+			}
+			paths[i] = "/" + strings.Join(segments, "/")
+		} else {
+			// Use the original mixed path generation logic for variable length paths
+			switch rand.Intn(6) {
+			case 0:
+				paths[i] = "/users/profile/settings"
+			case 1:
+				paths[i] = fmt.Sprintf("/users/%d/profile", i%200)
+			case 2:
+				paths[i] = fmt.Sprintf("/api/v1/users/%d/posts/%d", i%200, i%150)
+			case 3:
+				paths[i] = fmt.Sprintf("/organizations/%d/departments/%d/employees/%d/projects/%d/tasks/%d",
+					i%100, i%50, i%1000, i%30, i%200)
+			case 4:
+				repeatedSegment := fmt.Sprintf("%d", i%150)
+				paths[i] = fmt.Sprintf("/categories/%s/subcategories/%s/items/%s",
+					repeatedSegment, repeatedSegment, repeatedSegment)
+			case 5:
+				paths[i] = fmt.Sprintf("/articles/%d/%s-%s-%s",
+					i%100,
+					generateRandomString(5),
+					generateRandomString(7),
+					generateRandomString(5))
+			}
+		}
+	}
+	return paths
+}
+
+// Helper function to generate random strings
+func generateRandomString(length int) string {
+	const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+	result := make([]byte, length)
+	for i := range result {
+		result[i] = charset[rand.Intn(len(charset))]
+	}
+	return string(result)
+}