AV-203742 implement multitenancy in AMKO

go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/openshift/api v3.9.0+incompatible
 	github.com/openshift/client-go v0.0.0-20201020082437-7737f16e53fc
 	github.com/prometheus/client_golang v1.17.0
-	github.com/vmware/alb-sdk v0.0.0-20240422063246-6f25c71c5791
-	github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240426075151-cf4b379fde72
+	github.com/vmware/alb-sdk v0.0.0-20240619053936-3103500da639
+	github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240621074433-756d124a591e
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -460,10 +460,10 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vmware-tanzu/service-apis v0.0.0-20200901171416-461d35e58618 h1:ZLkxMxSr/YrYENjUJ8g6uB4rUQpZKBR3Olqf2DvuMFk=
 github.com/vmware-tanzu/service-apis v0.0.0-20200901171416-461d35e58618/go.mod h1:afqpDk9He9v+/qWix0RRotm3RNyni4Lmc1y9geDCPuo=
-github.com/vmware/alb-sdk v0.0.0-20240422063246-6f25c71c5791 h1:Qze7h4JtBPhRTRM+s2PxOF44grtjUMnfv4ELaWCmWsM=
-github.com/vmware/alb-sdk v0.0.0-20240422063246-6f25c71c5791/go.mod h1:fuRb4saDY/xy/UMeMvyKYmcplNknEL9ysaqYSw7reNE=
-github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240426075151-cf4b379fde72 h1:IjR5q7PdWnMnUh7pHBnjoWbtAYWGFWb1D0HneIXj0Ho=
-github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240426075151-cf4b379fde72/go.mod h1:QMSAftQsMjzYomBl2+90VedAzZdyj6hzQAJKlJBwhyA=
+github.com/vmware/alb-sdk v0.0.0-20240619053936-3103500da639 h1:UaTxFP5LeiSOlLEyHC9kmYuzdZWmvYpPB7jevg+ASNw=
+github.com/vmware/alb-sdk v0.0.0-20240619053936-3103500da639/go.mod h1:fuRb4saDY/xy/UMeMvyKYmcplNknEL9ysaqYSw7reNE=
+github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240621074433-756d124a591e h1:9BXPQ9VrftHpLruVgL/MInIJ2JgOOmn3pDul20386bQ=
+github.com/vmware/load-balancer-and-ingress-services-for-kubernetes v0.0.0-20240621074433-756d124a591e/go.mod h1:OYFWfOG6sHCDgYgMafq/Rcm3haP0q+9ZCUh0GGyVhCA=
 github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

gslb/cache/avi_ctrl.go

@@ -35,7 +35,7 @@ var aviClientInstance *utils.AviRestClientPool
 var clientOnce sync.Once
 
 // SharedAviClients initializes a pool of connections to the avi controller
-func SharedAviClients() *utils.AviRestClientPool {
+func SharedAviClients(tenant string) *utils.AviRestClientPool {
 	clientOnce.Do(func() {
 		var err error
 
@@ -44,7 +44,9 @@ func SharedAviClients() *utils.AviRestClientPool {
 			utils.AviLog.Fatal("AVI Controller information is missing, update them in kubernetes secret or via environment variable.")
 		}
 		os.Setenv("CTRL_VERSION", ctrlCfg.Version)
-		aviClientInstance, _, err = utils.NewAviRestClientPool(gslbutils.NumRestWorkers, ctrlCfg.IPAddr, ctrlCfg.Username, ctrlCfg.Password, "", ctrlCfg.Version, "")
+		userHeaders := utils.SharedCtrlProp().GetCtrlUserHeader()
+		apiScheme := utils.SharedCtrlProp().GetCtrlAPIScheme()
+		aviClientInstance, _, err = utils.NewAviRestClientPool(gslbutils.NumRestWorkers, ctrlCfg.IPAddr, ctrlCfg.Username, ctrlCfg.Password, "", ctrlCfg.Version, "", tenant, apiScheme, userHeaders)
 		if err != nil {
 			utils.AviLog.Errorf("AVI Controller Initialization failed, %s", err)
 		}
@@ -53,7 +55,7 @@ func SharedAviClients() *utils.AviRestClientPool {
 }
 
 func IsAviSiteLeader() (bool, error) {
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("admin")
 	if len(aviRestClientPool.AviClient) < 1 {
 		gslbutils.Errf("no avi clients initialized, returning")
 		return false, errors.New("no avi clients initialized")
@@ -181,8 +183,8 @@ func GetGslbLeaderUuid(client *clients.AviClient) (string, error) {
 	return leaderUUID, nil
 }
 
-func GetHMFromName(name string, gdp bool) (*models.HealthMonitor, error) {
-	aviClient := SharedAviClients().AviClient[0]
+func GetHMFromName(name string, gdp bool, tenant string) (*models.HealthMonitor, error) {
+	aviClient := SharedAviClients(tenant).AviClient[0]
 	uri := "api/healthmonitor?name=" + name
 
 	result, err := gslbutils.GetUriFromAvi(uri, aviClient, gdp)

gslb/cache/controller_obj_cache.go

@@ -205,7 +205,7 @@ func (h *AviHmCache) AviHmObjCachePopulate(client *clients.AviClient, hmname ...
 				continue
 			}
 
-			k := TenantName{Tenant: gslbutils.GetTenant(), Name: *hm.Name}
+			k := TenantName{Tenant: getTenantFromTenantRef(*hm.TenantRef), Name: *hm.Name}
 			var monitorPort int32
 			if hm.MonitorPort != nil {
 				monitorPort = *hm.MonitorPort
@@ -234,7 +234,7 @@ func (h *AviHmCache) AviHmObjCachePopulate(client *clients.AviClient, hmname ...
 			cksum := gslbutils.GetGSLBHmChecksum(*hm.Type, monitorPort, []string{description}, createdBy)
 			hmCacheObj := AviHmObj{
 				Name:             *hm.Name,
-				Tenant:           gslbutils.GetTenant(),
+				Tenant:           getTenantFromTenantRef(*hm.TenantRef),
 				UUID:             *hm.UUID,
 				Port:             monitorPort,
 				CloudConfigCksum: cksum,
@@ -342,7 +342,7 @@ func (s *AviPkiCache) AviPkiCachePopulate(client *clients.AviClient) error {
 				continue
 			}
 
-			k := TenantName{Tenant: gslbutils.GetTenant(), Name: *sp.Name}
+			k := TenantName{Tenant: getTenantFromTenantRef(*sp.TenantRef), Name: *sp.Name}
 			s.AviPkiCacheAdd(k, &sp)
 			s.AviPkiCacheAddByUUID(*sp.UUID, &sp)
 			gslbutils.Debugf("processed pki profile %s, UUID: %s", *sp.Name, *sp.UUID)
@@ -443,7 +443,7 @@ func (s *AviSpCache) AviSitePersistenceCachePopulate(client *clients.AviClient)
 				continue
 			}
 
-			k := TenantName{Tenant: gslbutils.GetTenant(), Name: *sp.Name}
+			k := TenantName{Tenant: getTenantFromTenantRef(*sp.TenantRef), Name: *sp.Name}
 			s.AviSpCacheAdd(k, &sp)
 			s.AviSpCacheAddByUUID(*sp.UUID, &sp)
 			gslbutils.Debugf("processed site persistence %s, UUID: %s", *sp.Name, *sp.UUID)
@@ -636,10 +636,10 @@ func parseGSObject(c *AviCache, gsObj models.GslbService, gsname []string) {
 			return
 		}
 	}
-	k := TenantName{Tenant: gslbutils.GetTenant(), Name: name}
+	k := TenantName{Tenant: getTenantFromTenantRef(*gsObj.TenantRef), Name: name}
 	gsCacheObj := AviGSCache{
 		Name:             name,
-		Tenant:           gslbutils.GetTenant(),
+		Tenant:           getTenantFromTenantRef(*gsObj.TenantRef),
 		Uuid:             uuid,
 		Members:          gsMembers,
 		K8sObjects:       memberObjs,
@@ -955,8 +955,8 @@ func GetDetailsFromAviGSLBFormatted(gsObj models.GslbService) (uint32, []GSMembe
 
 // As name is encoded, retreiving information about the Hm becomes difficult
 // Thats why we fetch Hm description for further processing
-func GetHmDescriptionFromName(hmName string) string {
-	aviRestClientPool := SharedAviClients()
+func GetHmDescriptionFromName(hmName, tenant string) string {
+	aviRestClientPool := SharedAviClients(tenant)
 	if len(aviRestClientPool.AviClient) < 1 {
 		return ""
 	}
@@ -1010,8 +1010,8 @@ func GetGSNameFromHmName(hmName string) (string, error) {
 // gen = 1 if the HM name is encoded and the gsname is fetched from its description
 // gen = 2 if the HM name follows the old non encoded naming convention and gsname is fetched from hmname
 // gen = 0 for error
-func GetGSFromHmName(hmName string) (string, int8, error) {
-	hmDesc := GetHmDescriptionFromName(hmName)
+func GetGSFromHmName(hmName, tenant string) (string, int8, error) {
+	hmDesc := GetHmDescriptionFromName(hmName, tenant)
 	hmDescriptionSplit := strings.Split(hmDesc, "gsname: ")
 	if len(hmDescriptionSplit) == 2 {
 		gsNameField := strings.Split(hmDescriptionSplit[1], ",")
@@ -1210,7 +1210,7 @@ type TenantName struct {
 }
 
 func PopulateGSCache(createSharedCache bool) *AviCache {
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("*")
 	var aviObjCache *AviCache
 	if createSharedCache {
 		aviObjCache = GetAviCache()
@@ -1228,7 +1228,7 @@ func PopulateGSCache(createSharedCache bool) *AviCache {
 }
 
 func PopulateHMCache(createSharedCache bool) *AviHmCache {
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("*")
 	var aviHmCache *AviHmCache
 	if createSharedCache {
 		aviHmCache = GetAviHmCache()
@@ -1245,7 +1245,7 @@ func PopulateHMCache(createSharedCache bool) *AviHmCache {
 }
 
 func PopulateSPCache() *AviSpCache {
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("*")
 	aviSpCache := GetAviSpCache()
 	if len(aviRestClientPool.AviClient) > 0 {
 		aviSpCache.AviSitePersistenceCachePopulate(aviRestClientPool.AviClient[0])
@@ -1254,7 +1254,7 @@ func PopulateSPCache() *AviSpCache {
 }
 
 func PopulatePkiCache() *AviPkiCache {
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("*")
 	aviSpCache := GetAviPkiCache()
 	if len(aviRestClientPool.AviClient) > 0 {
 		aviSpCache.AviPkiCachePopulate(aviRestClientPool.AviClient[0])
@@ -1266,7 +1266,7 @@ func VerifyVersion() error {
 	gslbutils.Logf("verifying the controller version")
 	version := gslbutils.GetAviConfig().Version
 
-	aviRestClientPool := SharedAviClients()
+	aviRestClientPool := SharedAviClients("*")
 	if len(aviRestClientPool.AviClient) < 1 {
 		gslbutils.Errf("no avi clients initialized, returning")
 		gslbutils.AMKOControlConfig().PodEventf(corev1.EventTypeWarning, gslbutils.AMKOShutdown, "No Avi clients initialized.")
@@ -1303,3 +1303,15 @@ func VerifyVersion() error {
 
 	return nil
 }
+
+func getTenantFromTenantRef(tenantRef string) string {
+	arr := strings.Split(tenantRef, "#")
+	if len(arr) == 2 {
+		return arr[1]
+	}
+	if len(arr) == 1 {
+		arr = strings.Split(tenantRef, "/")
+		return arr[len(arr)-1]
+	}
+	return tenantRef
+}

gslb/gslbutils/constants.go

@@ -53,10 +53,10 @@ const (
 	RejectedStore = "Rejected"
 
 	// Multi-cluster key lengths
-	IngMultiClusterKeyLen = 6
-	MultiClusterKeyLen    = 5
+	IngMultiClusterKeyLen = 7
+	MultiClusterKeyLen    = 6
 	GSFQDNKeyLen          = 3
-	HostRuleKeyLen        = 7
+	HostRuleKeyLen        = 8
 
 	// Default values for Retry Operations
 	SlowSyncTime      = 120

gslb/gslbutils/gslbutils.go

@@ -47,7 +47,13 @@ import (
 // InformersPerCluster is the number of informers per cluster
 var InformersPerCluster *utils.AviCache
 
+// LeaderClusterContext is the leader cluster where amko is deployed
+var LeaderClusterContext string
+
 func SetInformersPerCluster(clusterName string, info *utils.Informers) {
+	if InformersPerCluster == nil {
+		InformersPerCluster = utils.NewAviCache()
+	}
 	InformersPerCluster.AviCacheAdd(clusterName, info)
 }
 
@@ -60,8 +66,8 @@ func GetInformersPerCluster(clusterName string) *utils.Informers {
 	return info.(*utils.Informers)
 }
 
-func MultiClusterKey(operation, objType, clusterName, ns, objName string) string {
-	compositeObjName := clusterName + "/" + ns + "/" + objName
+func MultiClusterKey(operation, objType, clusterName, ns, objName, tenant string) string {
+	compositeObjName := clusterName + "/" + ns + "/" + objName + "/" + tenant
 	return MultiClusterKeyWithObjName(operation, objType, compositeObjName)
 }
 
@@ -73,8 +79,8 @@ func GSLBHostRuleKey(operation, objType, objName string) string {
 	return MultiClusterKeyWithObjName(operation, objType, objName)
 }
 
-func MultiClusterKeyForHostRule(operation, objType, clusterName, ns, objName, lfqdn, gfqdn string) string {
-	return MultiClusterKeyWithObjName(operation, objType, clusterName+"/"+ns+"/"+objName+"/"+lfqdn+"/"+gfqdn)
+func MultiClusterKeyForHostRule(operation, objType, clusterName, ns, objName, lfqdn, gfqdn, tenant string) string {
+	return MultiClusterKeyWithObjName(operation, objType, clusterName+"/"+ns+"/"+objName+"/"+lfqdn+"/"+gfqdn+"/"+tenant)
 }
 
 func ExtractMultiClusterHostRuleKey(key string) (string, string, string, string, string, string, error) {
@@ -85,33 +91,33 @@ func ExtractMultiClusterHostRuleKey(key string) (string, string, string, string,
 	return seg[0], seg[1], seg[2], seg[3], seg[4], seg[5], nil
 }
 
-func ExtractGSLBHostRuleKey(key string) (string, string, string, error) {
+func ExtractGSLBHostRuleKey(key string) (string, string, string, string, error) {
 	seg := strings.Split(key, "/")
-	if len(seg) != 3 {
-		return "", "", "", fmt.Errorf("invalid key %s for GSLBHostRule", key)
+	if len(seg) != 4 {
+		return "", "", "", "", fmt.Errorf("invalid key %s for GSLBHostRule", key)
 	}
-	return seg[0], seg[1], seg[2], nil
+	return seg[0], seg[1], seg[2], seg[3], nil
 }
 
-func ExtractMultiClusterKey(key string) (string, string, string, string, string) {
+func ExtractMultiClusterKey(key string) (string, string, string, string, string, string) {
 	segments := strings.Split(key, "/")
-	var operation, objType, cluster, ns, name, hostname string
+	var operation, objType, cluster, ns, name, hostname, tenant string
 	if segments[1] == IngressType ||
 		segments[1] == MCIType {
 		if len(segments) == IngMultiClusterKeyLen {
-			operation, objType, cluster, ns, name, hostname = segments[0], segments[1], segments[2], segments[3], segments[4], segments[5]
+			operation, objType, cluster, ns, name, hostname, tenant = segments[0], segments[1], segments[2], segments[3], segments[4], segments[5], segments[6]
 			name += "/" + hostname
 		} else if len(segments) == HostRuleKeyLen {
-			operation, objType, cluster, ns, name = segments[0], segments[1], segments[2], segments[3], segments[4]+"/"+segments[5]
+			operation, objType, cluster, ns, name, tenant = segments[0], segments[1], segments[2], segments[3], segments[4]+"/"+segments[5], segments[7]
 		}
 	} else if len(segments) == MultiClusterKeyLen {
-		operation, objType, cluster, ns, name = segments[0], segments[1], segments[2], segments[3], segments[4]
+		operation, objType, cluster, ns, name, tenant = segments[0], segments[1], segments[2], segments[3], segments[4], segments[5]
 	} else if len(segments) == GSFQDNKeyLen {
 		operation, objType, name = segments[0], segments[1], segments[2]
 	} else if len(segments) == HostRuleKeyLen {
-		operation, objType, cluster, ns, name = segments[0], segments[1], segments[2], segments[3], segments[4]
+		operation, objType, cluster, ns, name, tenant = segments[0], segments[1], segments[2], segments[3], segments[4], segments[7]
 	}
-	return operation, objType, cluster, ns, name
+	return operation, objType, cluster, ns, name, tenant
 }
 
 // sname for ingress is ingName/hostname
@@ -127,8 +133,8 @@ func GetObjectTypeFromKey(key string) (string, error) {
 	return segments[1], nil
 }
 
-func GSFQDNKey(operation, objType, gsFqdn string) string {
-	return operation + "/" + objType + "/" + gsFqdn
+func GSFQDNKey(operation, objType, gsFqdn, namespace string) string {
+	return operation + "/" + objType + "/" + namespace + "/" + gsFqdn
 }
 
 func SplitMultiClusterObjectName(name string) (string, string, string, error) {
@@ -303,6 +309,10 @@ func GetAviAdminTenantRef() string {
 	return "https://" + os.Getenv("GSLB_CTRL_IPADDRESS") + "/api/tenant/?name=" + GetTenant()
 }
 
+func GetTenantRef(tenant string) string {
+	return "https://" + os.Getenv("GSLB_CTRL_IPADDRESS") + "/api/tenant/?name=" + tenant
+}
+
 // GSLBConfigObj is global and is initialized only once
 type GSLBConfigObj struct {
 	configObj  *gslbalphav1.GSLBConfig
@@ -447,6 +457,19 @@ func GetTenant() string {
 	return gslbLeaderConfig.Tenant
 }
 
+func GetTenantInNamespace(namespace, cname string) string {
+	nsObj, err := GetInformersPerCluster(cname).NSInformer.Lister().Get(namespace)
+	if err != nil {
+		utils.AviLog.Warnf("Failed to GET the namespace details falling back to the default tenant, namespace: %s, error :%s", namespace, err.Error())
+		return GetTenant()
+	}
+	tenant, ok := nsObj.Annotations[TenantAnnotation]
+	if !ok || tenant == "" {
+		return GetTenant()
+	}
+	return tenant
+}
+
 var allClusterContexts []string
 
 func AddClusterContext(cc string) {
@@ -743,3 +766,9 @@ func GetDefaultPKI(aviClient *clients.AviClient) *string {
 	}
 	return nil
 }
+
+const (
+	VSAnnotation         = "ako.vmware.com/host-fqdn-vs-uuid-map"
+	ControllerAnnotation = "ako.vmware.com/controller-cluster-uuid"
+	TenantAnnotation     = "ako.vmware.com/tenant"
+)

gslb/ingestion/bootup_handler.go

@@ -195,6 +195,7 @@ func HandleBootup(cfg *restclient.Config) (bool, error) {
 	if amkoCluster.Spec.IsLeader {
 		currentLeader = true
 		gslbutils.Logf("AMKOCluster object found and AMKO would start as leader")
+		gslbutils.LeaderClusterContext = amkoCluster.Spec.ClusterContext
 	} else {
 		gslbutils.Logf("AMKOCluster object found and AMKO would start as follower")
 		return false, nil
@@ -213,7 +214,6 @@ func HandleBootup(cfg *restclient.Config) (bool, error) {
 		gslbutils.Warnf("some member cluster contexts couldn't be fetched: %s, will ignore these", federator.GetClusterErrMsg(errClusters))
 	}
 	gslbutils.Logf("memberClusters list found from amkoCluster object: %v", memberClusters)
-
 	_, errClusters, err = federator.ValidateMemberClusters(context.TODO(), memberClusters, amkoCluster.Spec.Version)
 	if err != nil {
 		return false, fmt.Errorf("error in validating the member clusters: %v", err)