From 3f7c399de26ceccc65acc221c820307aa5779a86 Mon Sep 17 00:00:00 2001 From: breeze Date: Mon, 16 Nov 2020 22:03:07 +0800 Subject: [PATCH 1/2] optimize: add maxStalenessSeconds option when read preference with mongodb is not primary. --- src/storage/dal/mongo/local/mongo.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/storage/dal/mongo/local/mongo.go b/src/storage/dal/mongo/local/mongo.go index 4f9c4dd1ab..39c151dda2 100644 --- a/src/storage/dal/mongo/local/mongo.go +++ b/src/storage/dal/mongo/local/mongo.go @@ -975,6 +975,17 @@ func validHostType(collection string, projection map[string]int, result interfac return nil } +const ( + // reference doc: + // https://docs.mongodb.com/manual/core/read-preference-staleness/#replica-set-read-preference-max-staleness + // this is the minimum value of maxStalenessSeconds allowed. + // specifying a smaller maxStalenessSeconds value will raise an error. Clients estimate secondaries’ staleness + // by periodically checking the latest write date of each replica set member. Since these checks are infrequent, + // the staleness estimate is coarse. Thus, clients cannot enforce a maxStalenessSeconds value of less than + // 90 seconds. + maxStalenessSeconds = 90 * time.Second +) + func getCollectionOption(ctx context.Context) *options.CollectionOptions { var opt *options.CollectionOptions switch util.GetDBReadPreference(ctx) { @@ -987,19 +998,19 @@ func getCollectionOption(ctx context.Context) *options.CollectionOptions { } case common.PrimaryPreferredMode: opt = &options.CollectionOptions{ - ReadPreference: readpref.PrimaryPreferred(), + ReadPreference: readpref.PrimaryPreferred(readpref.WithMaxStaleness(maxStalenessSeconds)), } case common.SecondaryMode: opt = &options.CollectionOptions{ - ReadPreference: readpref.Secondary(), + ReadPreference: readpref.Secondary(readpref.WithMaxStaleness(maxStalenessSeconds)), } case common.SecondaryPreferredMode: opt = &options.CollectionOptions{ - ReadPreference: readpref.SecondaryPreferred(), + ReadPreference: readpref.SecondaryPreferred(readpref.WithMaxStaleness(maxStalenessSeconds)), } case common.NearestMode: opt = &options.CollectionOptions{ - ReadPreference: readpref.Nearest(), + ReadPreference: readpref.Nearest(readpref.WithMaxStaleness(maxStalenessSeconds)), } } From 34f7bd06f9ca8a2503a2a0d6930199f231e7740a Mon Sep 17 00:00:00 2001 From: breeze Date: Tue, 17 Nov 2020 12:00:43 +0800 Subject: [PATCH 2/2] =?UTF-8?q?optimize:=20=E4=BC=98=E5=8C=96=E6=9D=83?= =?UTF-8?q?=E9=99=90=E4=B8=AD=E5=BF=83=E6=8B=89=E5=8F=96=E4=B8=BB=E6=9C=BA?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E4=BF=A1=E6=81=AF=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../auth_server/logics/fetch_instance_info.go | 93 ++++++++++++------- .../auth_server/service/service.go | 2 - 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/scene_server/auth_server/logics/fetch_instance_info.go b/src/scene_server/auth_server/logics/fetch_instance_info.go index c3b409ca0e..2181938d5c 100644 --- a/src/scene_server/auth_server/logics/fetch_instance_info.go +++ b/src/scene_server/auth_server/logics/fetch_instance_info.go @@ -120,7 +120,9 @@ func (lgc *Logics) FetchInstanceInfo(kit *rest.Kit, resourceType iam.TypeID, fil } // fetch hosts' specified attributes info using host ids -func (lgc *Logics) FetchHostInfo(kit *rest.Kit, resourceType iam.TypeID, filter *types.FetchInstanceInfoFilter) ([]map[string]interface{}, error) { +func (lgc *Logics) FetchHostInfo(kit *rest.Kit, resourceType iam.TypeID, filter *types.FetchInstanceInfoFilter) ( + []map[string]interface{}, error) { + if resourceType != iam.Host { return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKResourceTypeField) } @@ -187,30 +189,56 @@ func (lgc *Logics) FetchHostInfo(kit *rest.Kit, resourceType iam.TypeID, filter hosts = append(hosts, hostArr...) } - // get cloud area for display name use - var cloudMap map[int64]string - var err error - if hasName { - cloudIDs := make([]int64, len(hosts)) - for index, host := range hosts { + if len(hosts) == 0 { + return hosts, nil + } + + cnt := len(hosts) + cloudIDList := make([]int64, cnt) + hostIDList := make([]int64, cnt) + + for index, host := range hosts { + if hasName { cloudID, err := util.GetInt64ByInterface(host[common.BKCloudIDField]) if err != nil { blog.Errorf("parse cloud area id failed, err: %v, host: %+v", err, host) return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKCloudIDField) } + cloudIDList[index] = cloudID + } - cloudIDs[index] = cloudID + hostID, err := util.GetInt64ByInterface(host[common.BKHostIDField]) + if err != nil { + blog.Errorf("parse host id failed, err: %v, host: %+v", err, host) + return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKHostIDField) } + hostIDList[index] = hostID + } - cloudMap, err = lgc.getCloudNameMapByIDs(kit, cloudIDs) + // get cloud area for display name use + var cloudMap map[int64]string + var err error + if hasName { + cloudMap, err = lgc.getCloudNameMapByIDs(kit, cloudIDList) if err != nil { return nil, err } } + hostPathMap, err := lgc.getHostIamPath(kit, resourceType, hostIDList) + if err != nil { + return nil, err + } + // covert id and display_name field for _, host := range hosts { - host[types.IDField] = util.GetStrByInterface(host[common.BKHostIDField]) + hostID, err := util.GetInt64ByInterface(host[common.BKHostIDField]) + if err != nil { + blog.Errorf("parse host id failed, err: %v, host: %+v", err, host) + return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKHostIDField) + } + // add id field + host[types.IDField] = strconv.FormatInt(hostID, 10) if hasName { cloudID, err := util.GetInt64ByInterface(host[common.BKCloudIDField]) @@ -218,18 +246,15 @@ func (lgc *Logics) FetchHostInfo(kit *rest.Kit, resourceType iam.TypeID, filter blog.Errorf("parse cloud area id failed, err: %v, host: %+v", err, host) return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKCloudIDField) } - - host[types.NameField] = getHostDisplayName(util.GetStrByInterface(host[common.BKHostInnerIPField]), cloudMap[cloudID]) + ip := util.GetStrByInterface(host[common.BKHostInnerIPField]) + host[types.NameField] = getHostDisplayName(ip, cloudMap[cloudID]) } if needPath { - host[sdktypes.IamPathKey], err = lgc.getHostIamPath(kit, resourceType, host) - if err != nil { - blog.ErrorJSON("getResourceIamPath failed, error: %s, instance: %s, rid: %s", err.Error(), host, kit.Rid) - return nil, err - } + host[sdktypes.IamPathKey] = hostPathMap[hostID] } } + return hosts, nil } @@ -261,17 +286,11 @@ func (lgc *Logics) getResourceIamPath(kit *rest.Kit, resourceType iam.TypeID, in return iamPath, nil } -func (lgc *Logics) getHostIamPath(kit *rest.Kit, resourceType iam.TypeID, host map[string]interface{}) ([]string, error) { +func (lgc *Logics) getHostIamPath(kit *rest.Kit, resourceType iam.TypeID, hostList []int64) (map[int64][]string, error) { if resourceType != iam.Host { return nil, kit.CCError.CCErrorf(common.CCErrCommParamsInvalid, common.BKResourceTypeField) } - hostID, err := util.GetInt64ByInterface(host[common.BKHostIDField]) - if err != nil { - blog.Errorf("hostID %v parse int failed, error: %s, rid: %s", host[common.BKHostIDField], err.Error(), kit.Rid) - return nil, err - } - // get host iam path, either in resource pool directory or in business TODO: support host in business module when topology is supported defaultBizID, err := lgc.GetResourcePoolBizID(kit) if err != nil { @@ -279,7 +298,7 @@ func (lgc *Logics) getHostIamPath(kit *rest.Kit, resourceType iam.TypeID, host m } req := &metadata.HostModuleRelationRequest{ - HostIDArr: []int64{hostID}, + HostIDArr: hostList, Fields: []string{common.BKHostIDField, common.BKAppIDField, common.BKSetIDField, common.BKModuleIDField}, Page: metadata.BasePage{ Limit: common.BKNoLimit, @@ -287,19 +306,21 @@ func (lgc *Logics) getHostIamPath(kit *rest.Kit, resourceType iam.TypeID, host m } res, err := lgc.CoreAPI.CoreService().Host().GetHostModuleRelation(kit.Ctx, kit.Header, req) if err != nil { - blog.Errorf("GetHostModuleRelation by host id %d failed, error: %s, rid: %s", hostID, err.Error(), kit.Rid) + blog.Errorf("GetHostModuleRelation by host id %v failed, err: %s, rid: %s", hostList, err, kit.Rid) return nil, err } + if !res.Result { - blog.Errorf("GetHostModuleRelation by host id %d failed, error code: %d, error message: %s, rid: %s", hostID, res.Code, res.ErrMsg, kit.Rid) + blog.Errorf("GetHostModuleRelation by host id %v failed, err: %v, rid: %s", hostList, + res.Code, res.ErrMsg, kit.Rid) return nil, res.CCError() } + if len(res.Data.Info) == 0 { - return nil, nil + return make(map[int64][]string), nil } - relationDistinctMap := make(map[string]bool) - iamPath := make([]string, 0) + relationMap := make(map[int64][]string) for _, relation := range res.Data.Info { var path string if relation.AppID == defaultBizID { @@ -307,10 +328,14 @@ func (lgc *Logics) getHostIamPath(kit *rest.Kit, resourceType iam.TypeID, host m } else { path = "/" + string(iam.Business) + "," + strconv.FormatInt(relation.AppID, 10) + "/" } - if !relationDistinctMap[path] { - relationDistinctMap[path] = true - iamPath = append(iamPath, path) + + if _, exist := relationMap[relation.HostID]; !exist { + relationMap[relation.HostID] = make([]string, 0) } + + relationMap[relation.HostID] = append(relationMap[relation.HostID], path) + } - return iamPath, nil + + return relationMap, nil } diff --git a/src/scene_server/auth_server/service/service.go b/src/scene_server/auth_server/service/service.go index f4176bd7fc..381eac55ea 100644 --- a/src/scene_server/auth_server/service/service.go +++ b/src/scene_server/auth_server/service/service.go @@ -30,7 +30,6 @@ import ( sdkauth "configcenter/src/scene_server/auth_server/sdk/auth" "configcenter/src/scene_server/auth_server/sdk/client" "configcenter/src/scene_server/auth_server/types" - "github.com/emicklei/go-restful" ) @@ -144,7 +143,6 @@ func setSupplierID(req *http.Request) { } req.Header.Set(common.BKHTTPOwnerID, sID) } - blog.V(4).Infof("final supplierID:%s", req.Header.Get(common.BKHTTPOwnerID)) } func (s *AuthService) WebService() *restful.Container {