From 112c8d439e64ba9b0e6618d102b67cb67f914918 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:15:25 +0800 Subject: [PATCH 01/13] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 49abf6d..6320b86 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ products: credential: access_key: // 必须, 云API的SecretId access_secret: // 必须, 云API的SecretKey - role: // 可选，可在云上CVM/TKE中使用，没有access_key和access_secret时会使用role申请临时密钥 region: // 必须, 实例所在区域信息 rate_limit: 15 // 腾讯云监控拉取指标数据限制, 官方默认限制最大20qps @@ -139,7 +138,6 @@ metrics: ```bash export TENCENTCLOUD_SECRET_ID="YOUR_ACCESS_KEY" export TENCENTCLOUD_SECRET_KEY="YOUR_ACCESS_SECRET" -export TENCENTCLOUD_SERVICE_ROLE = "YOUR_SERVICE_ROLE" export TENCENTCLOUD_REGION="REGION" ``` @@ -173,4 +171,3 @@ export TENCENTCLOUD_REGION="REGION" - From 457cec3e76fab0919e312253bfb4a8d17f9a1b5e Mon Sep 17 00:00:00 2001 From: zianazhao Date: Sun, 28 Aug 2022 19:38:08 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/instance/instance.go | 61 ++++++++++++++++++++++++++++++++- pkg/instance/repository_tdmq.go | 7 ++++ pkg/metric/label.go | 11 ++++-- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 2218bfe..42f3c94 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -3,6 +3,8 @@ package instance import ( "fmt" "reflect" + "regexp" + "unicode" ) // 每个产品的实例对象, 可用于配置导出指标的额外label填充, 根据字段名获取值 @@ -16,6 +18,9 @@ type TcInstance interface { // 根据字段名称获取该字段的值, 由各个产品接口具体实现 GetFieldValueByName(string) (string, error) + // 根据字段名称获取该字段的值, 由各个产品接口具体实现 + GetFieldValuesByName(string) (map[string][]string, error) + // 获取实例raw元数据, 每个实例类型不一样 GetMeta() interface{} } @@ -36,7 +41,7 @@ func (ins *baseTcInstance) GetMonitorQueryKey() string { func (ins *baseTcInstance) GetFieldValueByName(name string) (val string, err error) { defer func() { if err := recover(); err != nil { - //nothing ignore err + // nothing ignore err } }() v := ins.value.FieldByName(name) @@ -45,3 +50,57 @@ func (ins *baseTcInstance) GetFieldValueByName(name string) (val string, err err } return fmt.Sprintf("%v", v.Interface()), nil } + +func (ins *baseTcInstance) GetFieldValuesByName(name string) (val map[string][]string, err error) { + defer func() { + if err := recover(); err != nil { + // nothing ignore err + } + }() + v := ins.value.FieldByName(name) + if v.Kind() == reflect.Ptr { + v = reflect.Indirect(v) + } + valueMap := make(map[string][]string) + if v.Kind() == reflect.Slice { + for i := 0; i < v.Len(); i++ { + if v.Index(i).Elem().Kind() == reflect.String { + valueMap[name] = append(val[name], fmt.Sprintf("%v", v.Index(i).Elem().Interface())) + } else if v.Index(i).Elem().Kind() == reflect.Struct { + var tagKey, tagValue reflect.Value + + if v.Index(i).Elem().FieldByName("TagKey").IsValid() && v.Index(i).Elem().FieldByName("TagValue").IsValid() { + tagKey = v.Index(i).Elem().FieldByName("TagKey") + tagValue = v.Index(i).Elem().FieldByName("TagValue") + } else if v.Index(i).Elem().FieldByName("Key").IsValid() && v.Index(i).Elem().FieldByName("Value").IsValid() { + tagKey = v.Index(i).Elem().FieldByName("Key") + tagValue = v.Index(i).Elem().FieldByName("Value") + } + if tagKey.Kind() == reflect.Ptr { + tagKey = reflect.Indirect(tagKey) + } + if tagValue.Kind() == reflect.Ptr { + tagValue = reflect.Indirect(tagValue) + } + if IsValidTagKey(tagKey.String()) { + valueMap[tagKey.String()] = append(val[tagKey.String()], tagValue.String()) + } + } + } + } else { + valueMap[name] = append(val[name], fmt.Sprintf("%v", v.Interface())) + } + return valueMap, nil +} + +func IsValidTagKey(str string) bool { + for _, r := range str { + if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { + return false + } + } + if !regexp.MustCompile(`^[A-Za-z0-9_]+$`).MatchString(str) { + return false + } + return true +} diff --git a/pkg/instance/repository_tdmq.go b/pkg/instance/repository_tdmq.go index c39c7a0..0a0b0f9 100644 --- a/pkg/instance/repository_tdmq.go +++ b/pkg/instance/repository_tdmq.go @@ -18,6 +18,9 @@ func init() { registerRepository("QCE/TDMQ", NewTdmqTcInstanceRepository) } +var includeVip = "includeVip" +var includeVipTrue = "true" + type TdmqTcInstanceRepository struct { client *sdk.Client logger log.Logger @@ -29,6 +32,10 @@ func (repo *TdmqTcInstanceRepository) GetInstanceKey() string { func (repo *TdmqTcInstanceRepository) Get(id string) (instance TcInstance, err error) { req := sdk.NewDescribeRocketMQClustersRequest() + req.Filters = []*sdk.Filter{{ + Name: &includeVip, + Values: []*string{&includeVipTrue}, + }} req.ClusterIdList = []*string{&id} resp, err := repo.client.DescribeRocketMQClusters(req) if err != nil { diff --git a/pkg/metric/label.go b/pkg/metric/label.go index 3a40cb5..bb0926f 100644 --- a/pkg/metric/label.go +++ b/pkg/metric/label.go @@ -44,9 +44,14 @@ func (l *TcmLabels) GetValues(filters map[string]string, ins instance.TcInstance } } for _, name := range l.instanceLabelNames { - v, e := ins.GetFieldValueByName(name) - if e == nil && v != "" { - nameValues[name] = v + vMap, e := ins.GetFieldValuesByName(name) + if e == nil && vMap != nil { + for vName, values := range vMap { + for _, value := range values { + nameValues[vName] = value + } + + } } } for name, value := range l.constLabels { From 80ced700c548dd4f3b9f0cbac8823207cea7391a Mon Sep 17 00:00:00 2001 From: zianazhao Date: Tue, 30 Aug 2022 11:18:19 +0800 Subject: [PATCH 03/13] label-tag --- pkg/instance/instance.go | 18 ++---------------- pkg/util/label.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 pkg/util/label.go diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 42f3c94..330b920 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -2,9 +2,8 @@ package instance import ( "fmt" + "github.com/tencentyun/tencentcloud-exporter/pkg/util" "reflect" - "regexp" - "unicode" ) // 每个产品的实例对象, 可用于配置导出指标的额外label填充, 根据字段名获取值 @@ -68,7 +67,6 @@ func (ins *baseTcInstance) GetFieldValuesByName(name string) (val map[string][]s valueMap[name] = append(val[name], fmt.Sprintf("%v", v.Index(i).Elem().Interface())) } else if v.Index(i).Elem().Kind() == reflect.Struct { var tagKey, tagValue reflect.Value - if v.Index(i).Elem().FieldByName("TagKey").IsValid() && v.Index(i).Elem().FieldByName("TagValue").IsValid() { tagKey = v.Index(i).Elem().FieldByName("TagKey") tagValue = v.Index(i).Elem().FieldByName("TagValue") @@ -82,7 +80,7 @@ func (ins *baseTcInstance) GetFieldValuesByName(name string) (val map[string][]s if tagValue.Kind() == reflect.Ptr { tagValue = reflect.Indirect(tagValue) } - if IsValidTagKey(tagKey.String()) { + if util.IsValidTagKey(tagKey.String()) { valueMap[tagKey.String()] = append(val[tagKey.String()], tagValue.String()) } } @@ -92,15 +90,3 @@ func (ins *baseTcInstance) GetFieldValuesByName(name string) (val map[string][]s } return valueMap, nil } - -func IsValidTagKey(str string) bool { - for _, r := range str { - if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { - return false - } - } - if !regexp.MustCompile(`^[A-Za-z0-9_]+$`).MatchString(str) { - return false - } - return true -} diff --git a/pkg/util/label.go b/pkg/util/label.go new file mode 100644 index 0000000..95007b7 --- /dev/null +++ b/pkg/util/label.go @@ -0,0 +1,18 @@ +package util + +import ( + "regexp" + "unicode" +) + +func IsValidTagKey(str string) bool { + for _, r := range str { + if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { + return false + } + } + if !regexp.MustCompile(`^[A-Za-z0-9_]+$`).MatchString(str) { + return false + } + return true +} From d473d7bfb0849406531fb9d91e331ef471c899f1 Mon Sep 17 00:00:00 2001 From: zianazhao Date: Tue, 30 Aug 2022 15:39:47 +0800 Subject: [PATCH 04/13] =?UTF-8?q?fix=EF=BC=9ATDMQ=20RocketMQ=20=E7=89=88?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E4=B8=93=E4=BA=AB=E9=9B=86=E7=BE=A4=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/instance/repository_tdmq.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/instance/repository_tdmq.go b/pkg/instance/repository_tdmq.go index 0a0b0f9..7770f42 100644 --- a/pkg/instance/repository_tdmq.go +++ b/pkg/instance/repository_tdmq.go @@ -64,7 +64,10 @@ func (repo *TdmqTcInstanceRepository) ListByFilters(filters map[string]string) ( req.Offset = &offset req.Limit = &limit - + req.Filters = []*sdk.Filter{{ + Name: &includeVip, + Values: []*string{&includeVipTrue}, + }} getMoreInstances: resp, err := repo.client.DescribeRocketMQClusters(req) if err != nil { From 815f50a643e33064f01e5387cada89294445a558 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 11:46:50 +0800 Subject: [PATCH 05/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96ac939..3f0199e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ name: Docker on: push: - branches: [ master ] + branches: [ master,dev/newProduct ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: From 0c72062b3e0e728f8934748983315652b51710e2 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 11:53:51 +0800 Subject: [PATCH 06/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3f0199e..6706a7d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,7 +7,7 @@ name: Docker on: push: - branches: [ master,dev/newProduct ] + branches: [ master ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: @@ -39,9 +39,9 @@ jobs: # https://github.com/sigstore/cosign-installer - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422 + uses: sigstore/cosign-installer@main with: - cosign-release: 'v1.4.0' + cosign-release: 'v1.11.0' # Workaround: https://github.com/docker/build-push-action/issues/461 From 8c58353db6fc710199edf70abe96e5bb248197bc Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 12:52:33 +0800 Subject: [PATCH 07/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6706a7d..56e120b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -39,9 +39,9 @@ jobs: # https://github.com/sigstore/cosign-installer - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@main + uses: sigstore/cosign-installer@14d43345ff50608baaa37893f4822c406ed470a9 with: - cosign-release: 'v1.11.0' + cosign-release: 'v1.11.1' # Workaround: https://github.com/docker/build-push-action/issues/461 From 19eda3639a31495295d806b998f895133a20420b Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:30:42 +0800 Subject: [PATCH 08/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 56e120b..6706a7d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -39,9 +39,9 @@ jobs: # https://github.com/sigstore/cosign-installer - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@14d43345ff50608baaa37893f4822c406ed470a9 + uses: sigstore/cosign-installer@main with: - cosign-release: 'v1.11.1' + cosign-release: 'v1.11.0' # Workaround: https://github.com/docker/build-push-action/issues/461 From bf9818cdeeb5decac7d22496cba293f94aa1fcfe Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:38:07 +0800 Subject: [PATCH 09/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6706a7d..96ac939 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -39,9 +39,9 @@ jobs: # https://github.com/sigstore/cosign-installer - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@main + uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422 with: - cosign-release: 'v1.11.0' + cosign-release: 'v1.4.0' # Workaround: https://github.com/docker/build-push-action/issues/461 From 57767f78bf704947a2559cee5b17416440e307bc Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:43:34 +0800 Subject: [PATCH 10/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96ac939..56e120b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -39,9 +39,9 @@ jobs: # https://github.com/sigstore/cosign-installer - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422 + uses: sigstore/cosign-installer@14d43345ff50608baaa37893f4822c406ed470a9 with: - cosign-release: 'v1.4.0' + cosign-release: 'v1.11.1' # Workaround: https://github.com/docker/build-push-action/issues/461 From 7b2b47eb621b9a9490860a10ae0b9827dcff36fe Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:46:41 +0800 Subject: [PATCH 11/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 56e120b..e297fdf 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -70,7 +70,7 @@ jobs: # https://github.com/docker/build-push-action - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@v2 with: context: . push: ${{ github.event_name != 'pull_request' }} From 026efc09c5c7ca87a54147b343e9ee79fb29b970 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:52:54 +0800 Subject: [PATCH 12/13] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e297fdf..f2f338d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -70,7 +70,7 @@ jobs: # https://github.com/docker/build-push-action - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@965c6a410d446a30e95d35052c67d6eded60dad6 with: context: . push: ${{ github.event_name != 'pull_request' }} From 32789cce62319f1a7bf65c6104c6ecfa0b573b42 Mon Sep 17 00:00:00 2001 From: zianazhao <53551082+zianazhao@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:57:55 +0800 Subject: [PATCH 13/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6320b86..437f78f 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ products: ```yaml credential: access_key: // 必须, 云API的SecretId - access_secret: // 必须, 云API的SecretKey + secret_key: // 必须, 云API的SecretKey region: // 必须, 实例所在区域信息 rate_limit: 15 // 腾讯云监控拉取指标数据限制, 官方默认限制最大20qps