From 169680fedc444842938dcb617f7bc6a07320282a Mon Sep 17 00:00:00 2001 From: Evyatar Date: Tue, 10 Dec 2024 14:56:34 +0200 Subject: [PATCH 1/4] Adding support for wildcard dns intents in mapper --- src/mapper/pkg/dnscache/dns_cache.go | 12 ++++++ src/mapper/pkg/dnscache/dns_cache_test.go | 27 +++++++++++++ .../dns_intents_publisher.go | 40 +++++++++++++------ .../pkg/resolvers/schema.helpers.resolvers.go | 3 +- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/mapper/pkg/dnscache/dns_cache.go b/src/mapper/pkg/dnscache/dns_cache.go index 48e8104e..38059032 100644 --- a/src/mapper/pkg/dnscache/dns_cache.go +++ b/src/mapper/pkg/dnscache/dns_cache.go @@ -6,6 +6,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/viper" "net" + "strings" "time" ) @@ -38,6 +39,17 @@ func (d *DNSCache) GetResolvedIPs(dnsName string) []string { return entry } +func (d *DNSCache) GetMatchingEntriesForWildcard(dnsName string) []string { + result := make([]string, 0) + dnsSuffix := strings.ReplaceAll(dnsName, "*", "") // Strip the wildcard, leave the '.example.com' suffix + for entry := range d.cache.items { + if strings.HasSuffix(entry, dnsSuffix) { + result = append(result, entry) + } + } + return result +} + // CacheValue holds the value and its expiration time type CacheValue[V any] struct { Value V diff --git a/src/mapper/pkg/dnscache/dns_cache_test.go b/src/mapper/pkg/dnscache/dns_cache_test.go index 3e4f8880..b1629db2 100644 --- a/src/mapper/pkg/dnscache/dns_cache_test.go +++ b/src/mapper/pkg/dnscache/dns_cache_test.go @@ -83,6 +83,33 @@ func (s *DNSCacheTestSuite) TestTTL() { } +func (s *DNSCacheTestSuite) TestWildcardIP() { + cache := NewDNSCache() + cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) + dnsNames := cache.GetMatchingEntriesForWildcard("*.surf-forecast.com") + s.Require().Len(dnsNames, 1) + s.Require().Equal(dnsNames[0], "www.surf-forecast.com") + + resolvedIPs := cache.GetResolvedIPs(dnsNames[0]) + s.Require().Len(resolvedIPs, 1) + s.Require().Equal(resolvedIPs[0], IP1) +} + +func (s *DNSCacheTestSuite) TestMultipleWildcardIPs() { + cache := NewDNSCache() + cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) + cache.AddOrUpdateDNSData("api.surf-forecast.com", IP2, 60*time.Second) + dnsNames := cache.GetMatchingEntriesForWildcard("*.surf-forecast.com") + s.Require().Len(dnsNames, 2) + + resolvedIPs := make([]string, 0) + for _, dnsName := range dnsNames { + resolvedIPs = append(resolvedIPs, cache.GetResolvedIPs(dnsName)...) + } + s.Require().Len(resolvedIPs, 2) + s.Require().Equal(resolvedIPs, []string{IP1, IP2}) +} + func TestDNSCacheTestSuite(t *testing.T) { suite.Run(t, new(DNSCacheTestSuite)) } diff --git a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go index 9b90c2c7..ecf8c921 100644 --- a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go +++ b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go @@ -13,6 +13,7 @@ import ( "net" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "strings" "time" ) @@ -163,7 +164,11 @@ func (p *Publisher) compareIntentsAndStatus(clientIntents otterizev2alpha1.Clien } func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string]map[string]struct{}) bool { - resolvedIPs := p.dnsCache.GetResolvedIPs(dnsName) + dnsNames := p.getDNSNameSlice(dnsName) + resolvedIPs := make([]string, 0) + for _, name := range dnsNames { + resolvedIPs = append(resolvedIPs, p.dnsCache.GetResolvedIPs(name)...) + } ips, ok := resolvedIPsMap[dnsName] if !ok { @@ -171,20 +176,21 @@ func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string] } if len(resolvedIPs) == 0 { - // Try to resolve it ourselves - ctxTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctxTimeout, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - logrus.WithField("dnsName", dnsName).Warn("DNS cache miss, resolving it ourselves") - - ipaddrs, err := p.resolver.LookupIPAddr(ctxTimeout, dnsName) - if err != nil { - logrus.WithError(err).WithField("dnsName", dnsName).Error("Failed to resolve DNS") - return false - } + logrus.WithField("dnsName", dnsName).Debug("DNS cache miss, resolving it ourselves") + for _, name := range dnsNames { + // Try to resolve it ourselves + ipaddrs, err := p.resolver.LookupIPAddr(ctxTimeout, name) + if err != nil { + logrus.WithError(err).WithField("dnsName", name).Error("Failed to resolve DNS") + return false + } - for _, ip := range ipaddrs { - ips[ip.String()] = struct{}{} - p.dnsCache.AddOrUpdateDNSData(dnsName, ip.String(), 10*time.Second) + for _, ip := range ipaddrs { + ips[ip.String()] = struct{}{} + p.dnsCache.AddOrUpdateDNSData(name, ip.String(), 120*time.Second) + } } } else { logrus.WithField("dnsName", dnsName).Debug("DNS cache hit") @@ -200,3 +206,11 @@ func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string] resolvedIPsMap[dnsName] = ips return true } + +func (p *Publisher) getDNSNameSlice(dnsName string) []string { + result := []string{dnsName} + if !strings.HasPrefix(dnsName, "*") { + return result + } + return p.dnsCache.GetMatchingEntriesForWildcard(dnsName) +} diff --git a/src/mapper/pkg/resolvers/schema.helpers.resolvers.go b/src/mapper/pkg/resolvers/schema.helpers.resolvers.go index 3ffa3145..9d7e714d 100644 --- a/src/mapper/pkg/resolvers/schema.helpers.resolvers.go +++ b/src/mapper/pkg/resolvers/schema.helpers.resolvers.go @@ -198,11 +198,12 @@ func (r *Resolver) handleDNSCaptureResultsAsExternalTraffic(_ context.Context, d LastSeen: dest.LastSeen, DNSName: dest.Destination, } + ip := "(unknown)" if dest.DestinationIP != nil { ip = *dest.DestinationIP intent.IPs = map[externaltrafficholder.IP]struct{}{externaltrafficholder.IP(*dest.DestinationIP): {}} - ttl := 60 * time.Second + ttl := 300 * time.Second if dest.TTL != nil { ttl = time.Duration(*dest.TTL) * time.Second } From 5d94e5805d6b2c91c90bb21fc90b84d761fec9b0 Mon Sep 17 00:00:00 2001 From: Evyatar Date: Sun, 15 Dec 2024 13:38:53 +0200 Subject: [PATCH 2/4] CR fixes --- src/mapper/pkg/dnscache/dns_cache.go | 2 +- .../dns_intents_publisher.go | 40 +++++++++---------- .../pkg/resolvers/schema.helpers.resolvers.go | 2 +- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/mapper/pkg/dnscache/dns_cache.go b/src/mapper/pkg/dnscache/dns_cache.go index 38059032..c4b03eba 100644 --- a/src/mapper/pkg/dnscache/dns_cache.go +++ b/src/mapper/pkg/dnscache/dns_cache.go @@ -41,7 +41,7 @@ func (d *DNSCache) GetResolvedIPs(dnsName string) []string { func (d *DNSCache) GetMatchingEntriesForWildcard(dnsName string) []string { result := make([]string, 0) - dnsSuffix := strings.ReplaceAll(dnsName, "*", "") // Strip the wildcard, leave the '.example.com' suffix + dnsSuffix := strings.TrimPrefix(dnsName, "*") // Strip the wildcard, leave the '.example.com' suffix for entry := range d.cache.items { if strings.HasSuffix(entry, dnsSuffix) { result = append(result, entry) diff --git a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go index ecf8c921..e8aaf087 100644 --- a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go +++ b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go @@ -164,10 +164,14 @@ func (p *Publisher) compareIntentsAndStatus(clientIntents otterizev2alpha1.Clien } func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string]map[string]struct{}) bool { - dnsNames := p.getDNSNameSlice(dnsName) resolvedIPs := make([]string, 0) - for _, name := range dnsNames { - resolvedIPs = append(resolvedIPs, p.dnsCache.GetResolvedIPs(name)...) + if p.isWildcardDNS(dnsName) { + matchingDNSNames := p.dnsCache.GetMatchingEntriesForWildcard(dnsName) + for _, name := range matchingDNSNames { + resolvedIPs = append(resolvedIPs, p.dnsCache.GetResolvedIPs(name)...) + } + } else { + resolvedIPs = p.dnsCache.GetResolvedIPs(dnsName) } ips, ok := resolvedIPsMap[dnsName] @@ -176,21 +180,19 @@ func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string] } if len(resolvedIPs) == 0 { - ctxTimeout, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctxTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() logrus.WithField("dnsName", dnsName).Debug("DNS cache miss, resolving it ourselves") - for _, name := range dnsNames { - // Try to resolve it ourselves - ipaddrs, err := p.resolver.LookupIPAddr(ctxTimeout, name) - if err != nil { - logrus.WithError(err).WithField("dnsName", name).Error("Failed to resolve DNS") - return false - } + // Try to resolve it ourselves + ipaddrs, err := p.resolver.LookupIPAddr(ctxTimeout, dnsName) + if err != nil { + logrus.WithError(err).WithField("dnsName", dnsName).Error("Failed to resolve DNS") + return false + } - for _, ip := range ipaddrs { - ips[ip.String()] = struct{}{} - p.dnsCache.AddOrUpdateDNSData(name, ip.String(), 120*time.Second) - } + for _, ip := range ipaddrs { + ips[ip.String()] = struct{}{} + p.dnsCache.AddOrUpdateDNSData(dnsName, ip.String(), 120*time.Second) } } else { logrus.WithField("dnsName", dnsName).Debug("DNS cache hit") @@ -207,10 +209,6 @@ func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string] return true } -func (p *Publisher) getDNSNameSlice(dnsName string) []string { - result := []string{dnsName} - if !strings.HasPrefix(dnsName, "*") { - return result - } - return p.dnsCache.GetMatchingEntriesForWildcard(dnsName) +func (p *Publisher) isWildcardDNS(dnsName string) bool { + return strings.HasPrefix(dnsName, "*") } diff --git a/src/mapper/pkg/resolvers/schema.helpers.resolvers.go b/src/mapper/pkg/resolvers/schema.helpers.resolvers.go index 9d7e714d..9709003a 100644 --- a/src/mapper/pkg/resolvers/schema.helpers.resolvers.go +++ b/src/mapper/pkg/resolvers/schema.helpers.resolvers.go @@ -203,7 +203,7 @@ func (r *Resolver) handleDNSCaptureResultsAsExternalTraffic(_ context.Context, d if dest.DestinationIP != nil { ip = *dest.DestinationIP intent.IPs = map[externaltrafficholder.IP]struct{}{externaltrafficholder.IP(*dest.DestinationIP): {}} - ttl := 300 * time.Second + ttl := 120 * time.Second if dest.TTL != nil { ttl = time.Duration(*dest.TTL) * time.Second } From 09e9d7fbd57eeb950bcaf681f9ff93b23c33de15 Mon Sep 17 00:00:00 2001 From: Evyatar Date: Sun, 15 Dec 2024 14:33:31 +0200 Subject: [PATCH 3/4] refactor & text fixes --- src/mapper/pkg/dnscache/dns_cache.go | 7 +++--- src/mapper/pkg/dnscache/dns_cache_test.go | 22 +++++-------------- .../dns_intents_publisher.go | 7 ++---- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/mapper/pkg/dnscache/dns_cache.go b/src/mapper/pkg/dnscache/dns_cache.go index c4b03eba..05284a47 100644 --- a/src/mapper/pkg/dnscache/dns_cache.go +++ b/src/mapper/pkg/dnscache/dns_cache.go @@ -39,12 +39,13 @@ func (d *DNSCache) GetResolvedIPs(dnsName string) []string { return entry } -func (d *DNSCache) GetMatchingEntriesForWildcard(dnsName string) []string { - result := make([]string, 0) +func (d *DNSCache) GetMatchingIPsForWildcard(dnsName string) []string { dnsSuffix := strings.TrimPrefix(dnsName, "*") // Strip the wildcard, leave the '.example.com' suffix + result := make([]string, 0) for entry := range d.cache.items { if strings.HasSuffix(entry, dnsSuffix) { - result = append(result, entry) + // Calling cache.Get() to utilize the LRU instead of iterating over the value too + result = append(result, d.cache.Get(entry)...) } } return result diff --git a/src/mapper/pkg/dnscache/dns_cache_test.go b/src/mapper/pkg/dnscache/dns_cache_test.go index b1629db2..5a8e3b81 100644 --- a/src/mapper/pkg/dnscache/dns_cache_test.go +++ b/src/mapper/pkg/dnscache/dns_cache_test.go @@ -86,28 +86,18 @@ func (s *DNSCacheTestSuite) TestTTL() { func (s *DNSCacheTestSuite) TestWildcardIP() { cache := NewDNSCache() cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) - dnsNames := cache.GetMatchingEntriesForWildcard("*.surf-forecast.com") - s.Require().Len(dnsNames, 1) - s.Require().Equal(dnsNames[0], "www.surf-forecast.com") - - resolvedIPs := cache.GetResolvedIPs(dnsNames[0]) - s.Require().Len(resolvedIPs, 1) - s.Require().Equal(resolvedIPs[0], IP1) + ips := cache.GetMatchingIPsForWildcard("*.surf-forecast.com") + s.Require().Len(ips, 1) + s.Require().Equal(ips[0], IP1) } func (s *DNSCacheTestSuite) TestMultipleWildcardIPs() { cache := NewDNSCache() cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) cache.AddOrUpdateDNSData("api.surf-forecast.com", IP2, 60*time.Second) - dnsNames := cache.GetMatchingEntriesForWildcard("*.surf-forecast.com") - s.Require().Len(dnsNames, 2) - - resolvedIPs := make([]string, 0) - for _, dnsName := range dnsNames { - resolvedIPs = append(resolvedIPs, cache.GetResolvedIPs(dnsName)...) - } - s.Require().Len(resolvedIPs, 2) - s.Require().Equal(resolvedIPs, []string{IP1, IP2}) + ips := cache.GetMatchingIPsForWildcard("*.surf-forecast.com") + s.Require().Len(ips, 2) + s.Require().Equal(ips, []string{IP1, IP2}) } func TestDNSCacheTestSuite(t *testing.T) { diff --git a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go index e8aaf087..02ea0df0 100644 --- a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go +++ b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go @@ -166,10 +166,7 @@ func (p *Publisher) compareIntentsAndStatus(clientIntents otterizev2alpha1.Clien func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string]map[string]struct{}) bool { resolvedIPs := make([]string, 0) if p.isWildcardDNS(dnsName) { - matchingDNSNames := p.dnsCache.GetMatchingEntriesForWildcard(dnsName) - for _, name := range matchingDNSNames { - resolvedIPs = append(resolvedIPs, p.dnsCache.GetResolvedIPs(name)...) - } + resolvedIPs = p.dnsCache.GetMatchingIPsForWildcard(dnsName) } else { resolvedIPs = p.dnsCache.GetResolvedIPs(dnsName) } @@ -179,7 +176,7 @@ func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string] ips = make(map[string]struct{}) } - if len(resolvedIPs) == 0 { + if len(resolvedIPs) == 0 && !p.isWildcardDNS(dnsName) { ctxTimeout, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() logrus.WithField("dnsName", dnsName).Debug("DNS cache miss, resolving it ourselves") From 2a39967f606ad6d024c5b652b1f485af8977f31a Mon Sep 17 00:00:00 2001 From: Evyatar Date: Sun, 15 Dec 2024 14:44:03 +0200 Subject: [PATCH 4/4] linter fix --- src/mapper/pkg/dnscache/dns_cache.go | 2 +- src/mapper/pkg/dnscache/dns_cache_test.go | 4 ++-- src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mapper/pkg/dnscache/dns_cache.go b/src/mapper/pkg/dnscache/dns_cache.go index 05284a47..f2c1c6fa 100644 --- a/src/mapper/pkg/dnscache/dns_cache.go +++ b/src/mapper/pkg/dnscache/dns_cache.go @@ -39,7 +39,7 @@ func (d *DNSCache) GetResolvedIPs(dnsName string) []string { return entry } -func (d *DNSCache) GetMatchingIPsForWildcard(dnsName string) []string { +func (d *DNSCache) GetResolvedIPsForWildcard(dnsName string) []string { dnsSuffix := strings.TrimPrefix(dnsName, "*") // Strip the wildcard, leave the '.example.com' suffix result := make([]string, 0) for entry := range d.cache.items { diff --git a/src/mapper/pkg/dnscache/dns_cache_test.go b/src/mapper/pkg/dnscache/dns_cache_test.go index 5a8e3b81..be5f2b5c 100644 --- a/src/mapper/pkg/dnscache/dns_cache_test.go +++ b/src/mapper/pkg/dnscache/dns_cache_test.go @@ -86,7 +86,7 @@ func (s *DNSCacheTestSuite) TestTTL() { func (s *DNSCacheTestSuite) TestWildcardIP() { cache := NewDNSCache() cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) - ips := cache.GetMatchingIPsForWildcard("*.surf-forecast.com") + ips := cache.GetResolvedIPsForWildcard("*.surf-forecast.com") s.Require().Len(ips, 1) s.Require().Equal(ips[0], IP1) } @@ -95,7 +95,7 @@ func (s *DNSCacheTestSuite) TestMultipleWildcardIPs() { cache := NewDNSCache() cache.AddOrUpdateDNSData("www.surf-forecast.com", IP1, 60*time.Second) cache.AddOrUpdateDNSData("api.surf-forecast.com", IP2, 60*time.Second) - ips := cache.GetMatchingIPsForWildcard("*.surf-forecast.com") + ips := cache.GetResolvedIPsForWildcard("*.surf-forecast.com") s.Require().Len(ips, 2) s.Require().Equal(ips, []string{IP1, IP2}) } diff --git a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go index 02ea0df0..b43531e7 100644 --- a/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go +++ b/src/mapper/pkg/dnsintentspublisher/dns_intents_publisher.go @@ -164,9 +164,9 @@ func (p *Publisher) compareIntentsAndStatus(clientIntents otterizev2alpha1.Clien } func (p *Publisher) appendResolvedIps(dnsName string, resolvedIPsMap map[string]map[string]struct{}) bool { - resolvedIPs := make([]string, 0) + var resolvedIPs []string if p.isWildcardDNS(dnsName) { - resolvedIPs = p.dnsCache.GetMatchingIPsForWildcard(dnsName) + resolvedIPs = p.dnsCache.GetResolvedIPsForWildcard(dnsName) } else { resolvedIPs = p.dnsCache.GetResolvedIPs(dnsName) }