From 6d4d3768b4880c8f465429c892c2dca8944c7982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Elnan?= Date: Wed, 24 Apr 2024 09:52:02 +0200 Subject: [PATCH 1/2] add metrics for userauth --- pkg/auth/authtools/tools.go | 15 +++++++++++++++ pkg/auth/userauth/activedirectory/ad.go | 11 ++++++++++- pkg/auth/userauth/ldaps/openldap.go | 16 ++++++++++++++-- pkg/auth/userauth/msgraph/msgraph.go | 12 +++++++++--- pkg/auth/userauth/userauth.go | 7 ++++--- 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/pkg/auth/authtools/tools.go b/pkg/auth/authtools/tools.go index fe365db5..b3681902 100644 --- a/pkg/auth/authtools/tools.go +++ b/pkg/auth/authtools/tools.go @@ -3,8 +3,23 @@ package authtools import ( "fmt" "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + UserLookupHistogram *prometheus.HistogramVec + ServerConnectionHistogram *prometheus.HistogramVec + ServerReconnectCounter *prometheus.CounterVec ) +func init() { + ServerConnectionHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{Name: "auth_server_connection_duration_seconds", Help: "Duration of server connection in seconds"}, []string{"provider", "domain", "host", "port", "status"}) + UserLookupHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{Name: "auth_user_lookup_duration_seconds", Help: "Duration of user lookup in seconds"}, []string{"provider", "domain", "status"}) + ServerReconnectCounter = promauto.NewCounterVec(prometheus.CounterOpts{Name: "auth_server_reconnects_total", Help: "Total number of server reconnects"}, []string{"provider", "domain"}) +} + func SplitUserId(userId string) (string, string, error) { parts := strings.Split(userId, "@") if len(parts) != 2 { diff --git a/pkg/auth/userauth/activedirectory/ad.go b/pkg/auth/userauth/activedirectory/ad.go index d5a9a501..8c2e5afe 100644 --- a/pkg/auth/userauth/activedirectory/ad.go +++ b/pkg/auth/userauth/activedirectory/ad.go @@ -1,6 +1,7 @@ package activedirectory import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -86,6 +87,7 @@ func (l *AdClient) Connect() error { for _, ldapserver := range l.config.Servers { rlog.Infof("Trying server %s for domain %s.", ldapserver.Host, l.config.Domain) + connectionStart := time.Now() if l.config.Certificate != nil { caCert := l.config.Certificate caCertPool := x509.NewCertPool() @@ -105,14 +107,17 @@ func (l *AdClient) Connect() error { if err != nil { rlog.Error("an error occurred connecting to LDAP-host.", err, rlog.Any("Host", ldapserver.Host), rlog.Any("Port", ldapserver.Port)) + authtools.ServerConnectionHistogram.WithLabelValues("ad", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "500").Observe(time.Since(connectionStart).Seconds()) continue } err = client.Bind(l.config.BindUser, l.config.BindPassword) if err != nil { rlog.Error("an error occurred authenticating to LDAP-host.", err, rlog.Any("Host", ldapserver.Host), rlog.Any("Port", ldapserver.Port), rlog.Any("BindUser", l.config.BindUser)) + authtools.ServerConnectionHistogram.WithLabelValues("ad", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "401").Observe(time.Since(connectionStart).Seconds()) } else { rlog.Infof("Connected to server server %s for domain %s.", ldapserver.Host, l.config.Domain) + authtools.ServerConnectionHistogram.WithLabelValues("ad", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "200").Observe(time.Since(connectionStart).Seconds()) break } } @@ -148,7 +153,7 @@ func (l *AdClient) search(basedn, filter string, attributes []string) (*ldap.Sea return nil, fmt.Errorf("could not fetch search entries") } -func (l *AdClient) GetUser(userId string) (*identitymodels.User, error) { +func (l *AdClient) GetUser(ctx context.Context, userId string) (*identitymodels.User, error) { userpart, domainpart, err := authtools.SplitUserId(userId) if err != nil { @@ -159,17 +164,21 @@ func (l *AdClient) GetUser(userId string) (*identitymodels.User, error) { if l.connection.IsClosing() { rlog.Debug("Reconnecting to Active Directory") + authtools.ServerReconnectCounter.WithLabelValues("ad", l.config.Domain).Inc() err := l.Connect() if err != nil { return nil, err } } + queryStart := time.Now() result, err := l.search(l.config.BaseDN, filter, attributes) if err != nil { + authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) return nil, err } + authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) var userEntry *ldap.Entry if result != nil && len(result.Entries) == 1 { for _, entry := range result.Entries { diff --git a/pkg/auth/userauth/ldaps/openldap.go b/pkg/auth/userauth/ldaps/openldap.go index 6e6be8c8..a2558c30 100644 --- a/pkg/auth/userauth/ldaps/openldap.go +++ b/pkg/auth/userauth/ldaps/openldap.go @@ -1,11 +1,13 @@ package ldaps import ( + "context" "crypto/tls" "crypto/x509" "errors" "fmt" "strconv" + "time" "github.com/NorskHelsenett/ror/pkg/auth/authtools" identitymodels "github.com/NorskHelsenett/ror/pkg/models/identity" @@ -51,6 +53,7 @@ func (l *LdapsClient) Connect() error { if err != nil { return fmt.Errorf("failed to parse default ldaps port") } + connectionStart := time.Now() if ldapserver.Port == ldapsport { caCert := l.config.Certificate caCertPool := x509.NewCertPool() @@ -70,15 +73,18 @@ func (l *LdapsClient) Connect() error { if err != nil { rlog.Error("an error occurred connecting to LDAP-host.", err, rlog.Any("Host", ldapserver.Host), rlog.Any("Port", ldapserver.Port)) + authtools.ServerConnectionHistogram.WithLabelValues("openldap", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "500").Observe(time.Since(connectionStart).Seconds()) } - err = client.Bind(l.config.BindUser, l.config.BindPassword) if err != nil { rlog.Error("an error occurred authenticating to LDAP-host.", err, rlog.Any("Host", ldapserver.Host), rlog.Any("Port", ldapserver.Port), rlog.Any("BindUser", l.config.BindUser)) + authtools.ServerConnectionHistogram.WithLabelValues("openldap", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "401").Observe(time.Since(connectionStart).Seconds()) } else { rlog.Infof("Connected to server server %s for domain %s.", ldapserver.Host, l.config.Domain) + authtools.ServerConnectionHistogram.WithLabelValues("openldap", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "200").Observe(time.Since(connectionStart).Seconds()) break } + } if client == nil { @@ -112,7 +118,7 @@ func (l *LdapsClient) search(basedn, filter string, attributes []string) (*ldap. return nil, fmt.Errorf("could not fetch search entries") } -func (l *LdapsClient) GetUser(userId string) (*identitymodels.User, error) { +func (l *LdapsClient) GetUser(ctx context.Context, userId string) (*identitymodels.User, error) { _, domainpart, err := authtools.SplitUserId(userId) if err != nil { @@ -126,17 +132,23 @@ func (l *LdapsClient) GetUser(userId string) (*identitymodels.User, error) { if l.connection.IsClosing() { rlog.Debug("Reconnecting to LDAP") + authtools.ServerReconnectCounter.WithLabelValues("openldap", l.config.Domain).Inc() err := l.Connect() if err != nil { return nil, err } } + queryStart := time.Now() result, err := l.search(l.config.BaseDN, filter, attributes) if err != nil { + authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) + return nil, err } + authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) + var userEntry *ldap.Entry if result != nil && len(result.Entries) == 1 { for _, entry := range result.Entries { diff --git a/pkg/auth/userauth/msgraph/msgraph.go b/pkg/auth/userauth/msgraph/msgraph.go index adc22275..dc560e02 100644 --- a/pkg/auth/userauth/msgraph/msgraph.go +++ b/pkg/auth/userauth/msgraph/msgraph.go @@ -5,6 +5,7 @@ package msgraph import ( "context" "fmt" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/NorskHelsenett/ror/pkg/auth/authtools" @@ -47,7 +48,7 @@ func NewMsGraphClient(config MsGraphConfig, cacheHelper kvcachehelper.CacheInter } else { client.GroupCache = memorycache.NewKvCache() } - + connectionStart := time.Now() cred, err := azidentity.NewClientSecretCredential(client.config.TenantID, client.config.ClientID, client.config.ClientSecret, nil) if err != nil { return nil, err @@ -56,9 +57,12 @@ func NewMsGraphClient(config MsGraphConfig, cacheHelper kvcachehelper.CacheInter conn, err := msgraphsdk.NewGraphServiceClientWithCredentials( cred, []string{"https://graph.microsoft.com/.default"}, ) + if err != nil { + authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, "https://graph.microsoft.com/.default", "443", "500").Observe(time.Since(connectionStart).Seconds()) return nil, err } + authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, "https://graph.microsoft.com/.default", "443", "200").Observe(time.Since(connectionStart).Seconds()) rlog.Infof("Connected to msgraph api for domain %s.", config.Domain) client.Client = conn return client, nil @@ -67,7 +71,7 @@ func NewMsGraphClient(config MsGraphConfig, cacheHelper kvcachehelper.CacheInter // GetUsersWithGroups gets a user and the name of the groups the user is a member of // TODO: Implement isExpired // TODO: Implement isDisabled... -func (g *MsGraphClient) GetUser(userId string) (*identitymodels.User, error) { +func (g *MsGraphClient) GetUser(ctx context.Context, userId string) (*identitymodels.User, error) { var ret *identitymodels.User var groupnames []string = []string{} var user models.Userable @@ -75,7 +79,7 @@ func (g *MsGraphClient) GetUser(userId string) (*identitymodels.User, error) { groupsChan := make(chan []string) userChan := make(chan models.Userable) errorChan := make(chan error) - + queryStart := time.Now() go g.getUser(userId, userChan, errorChan) go g.getGroups(userId, groupsChan, errorChan) @@ -90,11 +94,13 @@ func (g *MsGraphClient) GetUser(userId string) (*identitymodels.User, error) { case returneUser := <-userChan: user = returneUser case err := <-errorChan: + authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) return nil, err } } addDomainpartToGroups(&groupnames, userId) + authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) ret = &identitymodels.User{ Email: *user.GetUserPrincipalName(), diff --git a/pkg/auth/userauth/userauth.go b/pkg/auth/userauth/userauth.go index 51a33f13..6ff18197 100644 --- a/pkg/auth/userauth/userauth.go +++ b/pkg/auth/userauth/userauth.go @@ -1,6 +1,7 @@ package userauth import ( + "context" "encoding/json" "fmt" "strconv" @@ -26,7 +27,7 @@ type DomainResolverConfig struct { } type DomainResolverInterface interface { - GetUser(userId string) (*identitymodels.User, error) + GetUser(ctx context.Context, userId string) (*identitymodels.User, error) CheckHealth() []newhealth.Check } @@ -34,7 +35,7 @@ type DomainResolvers struct { resolvers map[string]DomainResolverInterface } -func (d DomainResolvers) GetUser(userId string) (*identitymodels.User, error) { +func (d DomainResolvers) GetUser(ctx context.Context, userId string) (*identitymodels.User, error) { _, domain, err := authtools.SplitUserId(userId) if err != nil { return nil, err @@ -45,7 +46,7 @@ func (d DomainResolvers) GetUser(userId string) (*identitymodels.User, error) { } if domainResolver, ok := d.resolvers[domain]; ok { - return domainResolver.GetUser(userId) + return domainResolver.GetUser(ctx, userId) } return nil, fmt.Errorf("no domain resolver found for domain: %s", domain) } From 530b5b6ef75f9b5f8df709d098dd8be08222c139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Elnan?= Date: Wed, 24 Apr 2024 10:29:10 +0200 Subject: [PATCH 2/2] add host to lookup metrics --- pkg/auth/authtools/tools.go | 2 +- pkg/auth/userauth/activedirectory/ad.go | 8 +++++--- pkg/auth/userauth/ldaps/openldap.go | 8 +++++--- pkg/auth/userauth/msgraph/msgraph.go | 12 +++++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/auth/authtools/tools.go b/pkg/auth/authtools/tools.go index b3681902..4729797b 100644 --- a/pkg/auth/authtools/tools.go +++ b/pkg/auth/authtools/tools.go @@ -16,7 +16,7 @@ var ( func init() { ServerConnectionHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{Name: "auth_server_connection_duration_seconds", Help: "Duration of server connection in seconds"}, []string{"provider", "domain", "host", "port", "status"}) - UserLookupHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{Name: "auth_user_lookup_duration_seconds", Help: "Duration of user lookup in seconds"}, []string{"provider", "domain", "status"}) + UserLookupHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{Name: "auth_user_lookup_duration_seconds", Help: "Duration of user lookup in seconds"}, []string{"provider", "domain", "host", "status"}) ServerReconnectCounter = promauto.NewCounterVec(prometheus.CounterOpts{Name: "auth_server_reconnects_total", Help: "Total number of server reconnects"}, []string{"provider", "domain"}) } diff --git a/pkg/auth/userauth/activedirectory/ad.go b/pkg/auth/userauth/activedirectory/ad.go index 3b115fb4..125bcc8c 100644 --- a/pkg/auth/userauth/activedirectory/ad.go +++ b/pkg/auth/userauth/activedirectory/ad.go @@ -22,7 +22,8 @@ import ( var DefaultTimeout = 10 * time.Second type AdConfig struct { - Domain string `json:"domain"` + Domain string `json:"domain"` + server string BindUser string `json:"bindUser"` BindPassword string `json:"bindPassword"` BaseDN string `json:"basedn"` @@ -121,6 +122,7 @@ func (l *AdClient) Connect() error { authtools.ServerConnectionHistogram.WithLabelValues("ad", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "401").Observe(time.Since(connectionStart).Seconds()) } else { rlog.Infof("Connected to server server %s for domain %s.", ldapserver.Host, l.config.Domain) + l.config.server = ldapserver.Host authtools.ServerConnectionHistogram.WithLabelValues("ad", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "200").Observe(time.Since(connectionStart).Seconds()) break } @@ -179,10 +181,10 @@ func (l *AdClient) GetUser(ctx context.Context, userId string) (*identitymodels. result, err := l.search(l.config.BaseDN, filter, attributes) if err != nil { - authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, l.config.server, "500").Observe(time.Since(queryStart).Seconds()) return nil, err } - authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("ad", l.config.Domain, l.config.server, "200").Observe(time.Since(queryStart).Seconds()) var userEntry *ldap.Entry if result != nil && len(result.Entries) == 1 { for _, entry := range result.Entries { diff --git a/pkg/auth/userauth/ldaps/openldap.go b/pkg/auth/userauth/ldaps/openldap.go index a4a1fd72..f0977d5f 100644 --- a/pkg/auth/userauth/ldaps/openldap.go +++ b/pkg/auth/userauth/ldaps/openldap.go @@ -19,7 +19,8 @@ import ( var DefaultTimeout = 10 * time.Second type LdapConfig struct { - Domain string `json:"domain"` + Domain string `json:"domain"` + server string BindUser string `json:"bindUser"` BindPassword string `json:"bindPassword"` BaseDN string `json:"basedn"` @@ -85,6 +86,7 @@ func (l *LdapsClient) Connect() error { authtools.ServerConnectionHistogram.WithLabelValues("openldap", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "401").Observe(time.Since(connectionStart).Seconds()) } else { rlog.Infof("Connected to server server %s for domain %s.", ldapserver.Host, l.config.Domain) + l.config.server = ldapserver.Host authtools.ServerConnectionHistogram.WithLabelValues("openldap", l.config.Domain, ldapserver.Host, strconv.Itoa(ldapserver.Port), "200").Observe(time.Since(connectionStart).Seconds()) break } @@ -147,11 +149,11 @@ func (l *LdapsClient) GetUser(ctx context.Context, userId string) (*identitymode result, err := l.search(l.config.BaseDN, filter, attributes) if err != nil { - authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, l.config.server, "500").Observe(time.Since(queryStart).Seconds()) return nil, err } - authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("openldap", l.config.Domain, l.config.server, "200").Observe(time.Since(queryStart).Seconds()) var userEntry *ldap.Entry if result != nil && len(result.Entries) == 1 { diff --git a/pkg/auth/userauth/msgraph/msgraph.go b/pkg/auth/userauth/msgraph/msgraph.go index dc560e02..e686e64e 100644 --- a/pkg/auth/userauth/msgraph/msgraph.go +++ b/pkg/auth/userauth/msgraph/msgraph.go @@ -19,6 +19,8 @@ import ( graphusers "github.com/microsoftgraph/msgraph-sdk-go/users" ) +var ApiEndpoint = "https://graph.microsoft.com/.default" + type MsGraphConfig struct { Domain string `json:"domain"` TenantID string `json:"tenantId"` @@ -55,14 +57,14 @@ func NewMsGraphClient(config MsGraphConfig, cacheHelper kvcachehelper.CacheInter } conn, err := msgraphsdk.NewGraphServiceClientWithCredentials( - cred, []string{"https://graph.microsoft.com/.default"}, + cred, []string{ApiEndpoint}, ) if err != nil { - authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, "https://graph.microsoft.com/.default", "443", "500").Observe(time.Since(connectionStart).Seconds()) + authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, ApiEndpoint, "443", "500").Observe(time.Since(connectionStart).Seconds()) return nil, err } - authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, "https://graph.microsoft.com/.default", "443", "200").Observe(time.Since(connectionStart).Seconds()) + authtools.ServerConnectionHistogram.WithLabelValues("msgraph", config.Domain, ApiEndpoint, "443", "200").Observe(time.Since(connectionStart).Seconds()) rlog.Infof("Connected to msgraph api for domain %s.", config.Domain) client.Client = conn return client, nil @@ -94,13 +96,13 @@ func (g *MsGraphClient) GetUser(ctx context.Context, userId string) (*identitymo case returneUser := <-userChan: user = returneUser case err := <-errorChan: - authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, "500").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, ApiEndpoint, "500").Observe(time.Since(queryStart).Seconds()) return nil, err } } addDomainpartToGroups(&groupnames, userId) - authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, "200").Observe(time.Since(queryStart).Seconds()) + authtools.UserLookupHistogram.WithLabelValues("msgraph", g.config.Domain, ApiEndpoint, "200").Observe(time.Since(queryStart).Seconds()) ret = &identitymodels.User{ Email: *user.GetUserPrincipalName(),