Do not load all groups. Verbose output

cmd/main.go

@@ -66,29 +66,20 @@ func main() {
 	if scimRecord == nil {
 		log.Fatal("SCIM record was not found. Make sure the record is valid and shared to KSM application")
 	}
-	var files = scimRecord.FindFiles("credentials.json")
-	var credentials = files[0].GetFileData()
-	var subject = scimRecord.GetFieldValueByType("login")
 
-	var fields = scimRecord.GetCustomFieldsByLabel("SCIM Group")
-	if len(fields) == 0 {
-		err = errors.New("\"SCIM Group\" custom field was not found. Please add a custom field \"SCIM Group\" to your record")
-		log.Println(err)
-		return
-	}
-	var scimGroups = scim.ParseScimGroups(fields)
-	if len(fields) == 0 {
-		err = errors.New("\"SCIM Group\" custom field does not contain any value")
+	var ka *scim.ScimEndpointParameters
+	var gcp *scim.GoogleEndpointParameters
+	if ka, gcp, err = scim.LoadScimParametersFromRecord(scimRecord); err != nil {
 		log.Println(err)
 		return
 	}
 
-	var googleEndpoint = scim.NewGoogleEndpoint(credentials, subject, scimGroups)
+	var googleEndpoint = scim.NewGoogleEndpoint(gcp.Credentials, gcp.AdminAccount, gcp.ScimGroups)
 
-	var scimUrl = scimRecord.GetFieldValueByType("url")
-	var token = scimRecord.Password()
+	var sync = scim.NewScimSync(googleEndpoint, ka.Url, ka.Token)
+	sync.SetVerbose(ka.Verbose)
+	sync.SetDestructive(ka.Destructive)
 
-	var sync = scim.NewScimSync(googleEndpoint, scimUrl, token)
 	var syncStat *scim.SyncStat
 	if syncStat, err = sync.Sync(); err != nil {
 		log.Fatal(err.Error())

scim/google_endpoint.go

@@ -3,9 +3,11 @@ package scim
 import (
 	"context"
 	"errors"
+	"fmt"
 	"golang.org/x/oauth2/google"
 	admin "google.golang.org/api/admin/directory/v1"
 	"google.golang.org/api/option"
+	"net/mail"
 	"strings"
 )
 
@@ -15,6 +17,7 @@ type googleEndpoint struct {
 	jwtCredentials []byte
 	subject        string
 	scimGroups     []string
+	logger         SyncDebugLogger
 }
 
 // NewGoogleEndpoint creates an ICrmDataSource for accessing Users and Groups in Google Workspace
@@ -28,6 +31,19 @@ func NewGoogleEndpoint(credentials []byte, subject string, scimGroups []string)
 		scimGroups:     scimGroups,
 	}
 }
+func (ge *googleEndpoint) DebugLogger() SyncDebugLogger {
+	if ge.logger != nil {
+		return ge.logger
+	}
+	return NilLogger
+}
+func (ge *googleEndpoint) SetDebugLogger(logger SyncDebugLogger) {
+	ge.logger = logger
+	if logger == nil {
+		ge.logger = NilLogger
+	} else {
+	}
+}
 
 func (ge *googleEndpoint) Users(cb func(*User)) {
 	if ge.users != nil {
@@ -45,6 +61,24 @@ func (ge *googleEndpoint) Groups(cb func(*Group)) {
 	}
 }
 
+func parseGoogleUser(gu *admin.User) (su *User) {
+	su = &User{
+		Id:     gu.Id,
+		Email:  gu.PrimaryEmail,
+		Active: !gu.Suspended,
+	}
+	if gu.Name != nil {
+		su.FirstName = gu.Name.GivenName
+		su.LastName = gu.Name.FamilyName
+		if len(gu.Name.FullName) > 0 {
+			su.FullName = gu.Name.FullName
+		} else {
+			su.FullName = strings.TrimSpace(strings.Join([]string{gu.Name.GivenName, gu.Name.FamilyName}, " "))
+		}
+	}
+	return
+}
+
 func (ge *googleEndpoint) Populate() (err error) {
 	params := google.CredentialsParams{
 		Scopes: []string{admin.AdminDirectoryUserReadonlyScope,
@@ -84,73 +118,48 @@ func (ge *googleEndpoint) Populate() (err error) {
 	}
 
 	ge.users = make(map[string]*User)
-	var userLookup = make(map[string]*User)
-	var firstRun = true
-	var nextToken string
-	for firstRun || len(nextToken) > 0 {
-		firstRun = false
-		var ul = directory.Users.List().Customer("my_customer")
-		if len(nextToken) > 0 {
-			ul = ul.PageToken(nextToken)
-		}
+	ge.groups = make(map[string]*Group)
 
-		var users *admin.Users
-		if users, err = ul.Do(); err == nil {
-			nextToken = users.NextPageToken
-			for _, u := range users.Users {
-				var gu = &User{
-					Id:     u.Id,
-					Email:  u.PrimaryEmail,
-					Active: !u.Suspended,
-				}
-				if u.Name != nil {
-					gu.FirstName = u.Name.GivenName
-					gu.LastName = u.Name.FamilyName
-					if len(u.Name.FullName) > 0 {
-						gu.FullName = u.Name.FullName
-					} else {
-						gu.FullName = strings.TrimSpace(strings.Join([]string{u.Name.GivenName, u.Name.FamilyName}, " "))
+	ge.DebugLogger()("Resolving \"SCIM Group\" content")
+	var users *admin.Users
+	var groups *admin.Groups
+	for entry := range scimGroups {
+		var address *mail.Address
+		if address, err = mail.ParseAddress(entry); err == nil {
+			var gl = directory.Groups.List().Customer("my_customer").Query(fmt.Sprintf("email=%s", address.Address))
+			if groups, err = gl.Do(); err == nil && len(groups.Groups) > 0 {
+				for _, g := range groups.Groups {
+					ge.DebugLogger()(fmt.Sprintf("Found Google group \"%s\" for email \"%s\"", g.Name, g.Email))
+					ge.groups[g.Id] = &Group{
+						Id:   g.Id,
+						Name: g.Name,
 					}
 				}
-				userLookup[gu.Id] = gu
-				if scimGroups.Has(strings.ToLower(gu.Email)) {
-					ge.users[gu.Id] = gu
+			} else {
+				var ul = directory.Users.List().Customer("my_customer").Query(fmt.Sprintf("email=%s", address.Address))
+				if users, err = ul.Do(); err == nil && len(users.Users) > 0 {
+					for _, u := range users.Users {
+						ge.DebugLogger()(fmt.Sprintf("Found Google user for email \"%s\"", u.PrimaryEmail))
+						var su = parseGoogleUser(u)
+						ge.users[su.Id] = su
+					}
+				} else {
+					ge.DebugLogger()(fmt.Sprintf("An email \"%s\" could not be resolved as either Google User or Group", address.Address))
 				}
 			}
 		} else {
-			err = errors.New("google directory API: error querying users")
-			return
-		}
-	}
-
-	ge.groups = make(map[string]*Group)
-	var groupLookup = make(map[string]*Group)
-	firstRun = true
-	nextToken = ""
-	for firstRun || len(nextToken) > 0 {
-		firstRun = false
-		var gl = directory.Groups.List().Customer("my_customer")
-		if len(nextToken) >= 0 {
-			gl = gl.PageToken(nextToken)
-		}
-		if gl != nil {
-			var groups *admin.Groups
-			if groups, err = gl.Do(); err == nil {
-				nextToken = groups.NextPageToken
+			var gl = directory.Groups.List().Customer("my_customer").Query(fmt.Sprintf("name='%s'", entry))
+			if groups, err = gl.Do(); err == nil && len(groups.Groups) > 0 {
 				for _, g := range groups.Groups {
-					var gg = &Group{
+					ge.DebugLogger()(fmt.Sprintf("Found Google group \"%s\" by name", g.Name))
+					ge.groups[g.Id] = &Group{
 						Id:   g.Id,
 						Name: g.Name,
 					}
-					groupLookup[gg.Id] = gg
-					if scimGroups.Has(strings.ToLower(g.Email)) || scimGroups.Has(strings.ToLower(g.Name)) {
-						ge.groups[gg.Id] = gg
-					}
 				}
+			} else {
+				ge.DebugLogger()(fmt.Sprintf("A name \"%s\" could not be resolved as Group. Names are case sensitive", address.Address))
 			}
-		} else {
-			err = errors.New("google directory API: error querying users")
-			return
 		}
 	}
 
@@ -159,10 +168,27 @@ func (ge *googleEndpoint) Populate() (err error) {
 		return
 	}
 
+	ge.DebugLogger()("Loading all users")
+	var userLookup = make(map[string]*User)
+	if err = directory.Users.List().Customer("my_customer").MaxResults(200).Pages(ctx, func(users *admin.Users) error {
+		var no = 0
+		for _, u := range users.Users {
+			var su = parseGoogleUser(u)
+			userLookup[su.Id] = su
+			no++
+		}
+		ge.DebugLogger()(fmt.Sprintf("User page contains %d element(s)", no))
+		return nil
+	}); err != nil {
+		err = errors.New("google directory API: error querying users")
+		return
+	}
+	ge.DebugLogger()(fmt.Sprintf("Total %d Google user(s) loaded", len(userLookup)))
+
 	var ok bool
 	// expand embedded groups
 	var membershipCache = make(map[string][]string)
-	for groupId := range ge.groups {
+	for groupId, group := range ge.groups {
 		var groupIds = []string{groupId}
 		var queuedIds = MakeSet[string](groupIds)
 		var pos = 0
@@ -172,12 +198,13 @@ func (ge *googleEndpoint) Populate() (err error) {
 
 			var memberIds []string
 			if memberIds, ok = membershipCache[gId]; !ok {
-				var members *admin.Members
-				if members, err = directory.Members.List(gId).Do(); err != nil {
-					return
-				}
-				for _, m := range members.Members {
-					memberIds = append(memberIds, m.Id)
+				if err = directory.Members.List(gId).Pages(ctx, func(members *admin.Members) error {
+					for _, m := range members.Members {
+						memberIds = append(memberIds, m.Id)
+					}
+					return nil
+				}); err != nil {
+					ge.DebugLogger()(fmt.Sprintf("Loaded group \"%s\" membership failed: %s", group.Name, err.Error()))
 				}
 				membershipCache[gId] = memberIds
 			}
@@ -189,7 +216,7 @@ func (ge *googleEndpoint) Populate() (err error) {
 					if _, ok = ge.users[u.Id]; !ok {
 						ge.users[u.Id] = u
 					}
-				} else if g, ok = groupLookup[mId]; ok {
+				} else {
 					if !queuedIds.Has(g.Id) {
 						groupIds = append(groupIds, g.Id)
 						queuedIds.Add(g.Id)

scim/ksm_utils.go

@@ -0,0 +1,51 @@
+package scim
+
+import (
+	"errors"
+	ksm "github.com/keeper-security/secrets-manager-go/core"
+)
+
+func LoadScimParametersFromRecord(scimRecord *ksm.Record) (ka *ScimEndpointParameters, gcp *GoogleEndpointParameters, err error) {
+	var files = scimRecord.FindFiles("credentials.json")
+	var credentials = files[0].GetFileData()
+	var subject = scimRecord.GetFieldValueByType("login")
+
+	var fields = scimRecord.GetCustomFieldsByLabel("SCIM Group")
+	if len(fields) == 0 {
+		err = errors.New("\"SCIM Group\" custom field was not found. Please add a custom field \"SCIM Group\" to your record")
+		return
+	}
+	var scimGroups = ParseScimGroups(fields)
+	if len(scimGroups) == 0 {
+		err = errors.New("\"SCIM Group\" custom field does not contain any value")
+		return
+	}
+
+	gcp = &GoogleEndpointParameters{
+		AdminAccount: subject,
+		Credentials:  credentials,
+		ScimGroups:   scimGroups,
+	}
+
+	ka = &ScimEndpointParameters{
+		Url:   scimRecord.GetFieldValueByType("url"),
+		Token: scimRecord.Password(),
+	}
+
+	var ok bool
+	var bv bool
+	fields = scimRecord.GetCustomFieldsByLabel("Verbose")
+	if len(fields) > 0 {
+		if bv, ok = toBoolean(fields[0]["value"]); ok {
+			ka.Verbose = bv
+		}
+	}
+
+	fields = scimRecord.GetCustomFieldsByLabel("Destructive")
+	if len(fields) > 0 {
+		if bv, ok = toBoolean(fields[0]["value"]); ok {
+			ka.Destructive = bv
+		}
+	}
+	return
+}

scim/scim_data.go

@@ -1,9 +1,15 @@
 package scim
 
+type SyncDebugLogger func(string)
+
+var NilLogger SyncDebugLogger = func(string) {}
+
 type ICrmDataSource interface {
 	Users(func(*User))
 	Groups(func(*Group))
 	Populate() error
+	DebugLogger() SyncDebugLogger
+	SetDebugLogger(SyncDebugLogger)
 }
 
 type SyncStat struct {
@@ -17,6 +23,10 @@ type SyncStat struct {
 type IScimSync interface {
 	Source() ICrmDataSource
 	Sync() (*SyncStat, error)
+	Verbose() bool
+	SetVerbose(bool)
+	Destructive() bool
+	SetDestructive(bool)
 }
 
 type User struct {
@@ -33,3 +43,16 @@ type Group struct {
 	Id   string
 	Name string
 }
+
+type ScimEndpointParameters struct {
+	Url         string
+	Token       string
+	Verbose     bool
+	Destructive bool
+}
+
+type GoogleEndpointParameters struct {
+	AdminAccount string
+	Credentials  []byte
+	ScimGroups   []string
+}