test: 提高单测覆盖度 (#26)

pkg/collector/control.sh

@@ -14,7 +14,7 @@ set -e
 # shellcheck disable=SC2086
 
 MODULE=bk-collector
-TEST_COVERAGE_THRESHOLD=80
+TEST_COVERAGE_THRESHOLD=82
 
 function unittest() {
   go test ./... -coverprofile coverage.out -covermode count

pkg/collector/define/generator.go

@@ -9,7 +9,9 @@
 
 package define
 
-import "time"
+import (
+	"time"
+)
 
 type GeneratorOptions struct {
 	Enabled             bool

pkg/collector/go.mod

@@ -112,7 +112,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.7.4-0.20211011103944-1a7a2bd3279f // indirect
 	github.com/prometheus/prometheus v0.37.0 // indirect
-	github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

pkg/collector/go.sum

@@ -547,8 +547,6 @@ github.com/prometheus/procfs v0.7.4-0.20211011103944-1a7a2bd3279f/go.mod h1:cz+a
 github.com/prometheus/prometheus v0.37.0 h1:LgnE+97wnUK/qcmk5oHIqieJEKwhZtaSidyKpUyeats=
 github.com/prometheus/prometheus v0.37.0/go.mod h1:egARUgz+K93zwqsVIAneFlLZefyGOON44WyAp4Xqbbk=
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
-github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc h1:eXQoy66wUI9meNnIdKYJ+EV/Tq3LvXeUe95AB2dPk8g=
-github.com/rapidloop/skv v0.0.0-20180909015525-9def2caac4cc/go.mod h1:V5hvlcTzUJ3MOo0fEolWR25CZBBsb7q3wWVAmBYwr54=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=

pkg/collector/internal/cleaner/cleaner.go

@@ -9,7 +9,9 @@
 
 package cleaner
 
-import "sync"
+import (
+	"sync"
+)
 
 type CleanFunc func() error
 

pkg/collector/internal/json/json.go

@@ -9,7 +9,9 @@
 
 package json
 
-import "github.com/bytedance/sonic"
+import (
+	"github.com/bytedance/sonic"
+)
 
 var sonicAPI = sonic.Config{
 	EscapeHTML:       true, // 安全性

pkg/collector/internal/mapstructure/decode.go

@@ -9,7 +9,9 @@
 
 package mapstructure
 
-import "github.com/mitchellh/mapstructure"
+import (
+	"github.com/mitchellh/mapstructure"
+)
 
 // Decode 默认支持 time.Duration 类型数据的转换处理
 func Decode(input, output interface{}) error {

pkg/collector/processor/apdexcalculator/factory_test.go

@@ -42,26 +42,51 @@ processor:
           destination: "apdex_type"
           apdex_t: 20 # ms
 `
+	mainConf := processor.MustLoadConfigs(content)[0].Config
 
-	psc := processor.MustLoadConfigs(content)
-	obj, err := NewFactory(psc[0].Config, nil)
+	customContent := `
+processor:
+  - name: "apdex_calculator/standard"
+    config:
+      calculator:
+        type: "standard"
+      rules:
+        - kind: ""
+          metric_name: "bk_apm_duration"
+          destination: "apdex_type"
+          apdex_t: 50 # ms
+`
+	customConf := processor.MustLoadConfigs(customContent)[0].Config
+
+	obj, err := NewFactory(mainConf, []processor.SubConfigProcessor{
+		{
+			Token: "token1",
+			Type:  define.SubConfigFieldDefault,
+			Config: processor.Config{
+				Config: customConf,
+			},
+		},
+	})
 	factory := obj.(*apdexCalculator)
 	assert.NoError(t, err)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	assert.Equal(t, mainConf, factory.MainConfig())
 
-	var c Config
-	err = mapstructure.Decode(psc[0].Config, &c)
-	assert.NoError(t, err)
+	var c1 Config
+	assert.NoError(t, mapstructure.Decode(mainConf, &c1))
+	mainConfig := factory.configs.GetGlobal().(*Config)
+	assert.Equal(t, c1.Rules, mainConfig.Rules)
 
-	actualConfig := factory.configs.GetGlobal().(*Config)
-	assert.Equal(t, c.Rules, actualConfig.Rules)
+	var c2 Config
+	assert.NoError(t, mapstructure.Decode(customConf, &c2))
+	customConfig := factory.configs.GetByToken("token1").(*Config)
+	assert.Equal(t, c2.Rules, customConfig.Rules)
 
 	assert.Equal(t, define.ProcessorApdexCalculator, factory.Name())
 	assert.False(t, factory.IsDerived())
 	assert.False(t, factory.IsPreCheck())
 
-	factory.Reload(psc[0].Config, nil)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	factory.Reload(mainConf, nil)
+	assert.Equal(t, mainConf, factory.MainConfig())
 }
 
 func testMetricsDimension(t *testing.T, data interface{}, conf *Config, exist bool) {
@@ -250,6 +275,12 @@ func TestProcessTracesStandardCalculator(t *testing.T) {
 func testProcessTracesStandardCalculator(startTime, endTime time.Duration, threshold float64) (string, error) {
 	g := generator.NewTracesGenerator(define.TracesOptions{
 		SpanCount: 1,
+		GeneratorOptions: define.GeneratorOptions{
+			RandomAttributeKeys: []string{
+				processor.KeyService,
+				processor.KeyInstance,
+			},
+		},
 	})
 	data := g.Generate()
 	span := testkits.FirstSpan(data)

pkg/collector/processor/attributefilter/factory_test.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/define"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/generator"
-	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/mapstructure"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/testkits"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor"
 )
@@ -34,25 +33,43 @@ processor:
         keys:
           - "attributes.http.host"
 `
-	psc := processor.MustLoadConfigs(content)
-	obj, err := NewFactory(psc[0].Config, nil)
+	mainConf := processor.MustLoadConfigs(content)[0].Config
+
+	customContent := `
+processor:
+  - name: "attribute_filter/as_string"
+    config:
+      as_string:
+        keys:
+          - "attributes.http.port"
+`
+	customConf := processor.MustLoadConfigs(customContent)[0].Config
+
+	obj, err := NewFactory(mainConf, []processor.SubConfigProcessor{
+		{
+			Token: "token1",
+			Type:  define.SubConfigFieldDefault,
+			Config: processor.Config{
+				Config: customConf,
+			},
+		},
+	})
 	factory := obj.(*attributeFilter)
 	assert.NoError(t, err)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	assert.Equal(t, mainConf, factory.MainConfig())
 
-	var c Config
-	err = mapstructure.Decode(psc[0].Config, &c)
-	assert.NoError(t, err)
+	mainConfig := factory.configs.GetGlobal().(Config)
+	assert.Equal(t, "http.host", mainConfig.AsString.Keys[0])
 
-	c.AsString.Keys[0] = "http.host"
-	assert.Equal(t, c, factory.configs.GetGlobal().(Config))
+	customConfig := factory.configs.GetByToken("token1").(Config)
+	assert.Equal(t, "http.port", customConfig.AsString.Keys[0])
 
 	assert.Equal(t, define.ProcessorAttributeFilter, factory.Name())
 	assert.False(t, factory.IsDerived())
 	assert.False(t, factory.IsPreCheck())
 
-	factory.Reload(psc[0].Config, nil)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	factory.Reload(mainConf, nil)
+	assert.Equal(t, mainConf, factory.MainConfig())
 }
 
 const (

pkg/collector/processor/dbfilter/config.go

@@ -9,7 +9,9 @@
 
 package dbfilter
 
-import "time"
+import (
+	"time"
+)
 
 type Config struct {
 	SlowQuery SlowQueryConfig `config:"slow_query" mapstructure:"slow_query"`

pkg/collector/processor/dbfilter/factory_test.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/define"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/generator"
-	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/mapstructure"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/internal/testkits"
 	"github.com/TencentBlueKing/bkmonitor-datalink/pkg/collector/processor"
 )
@@ -38,29 +37,45 @@ processor:
         - match: "redis"
           threshold: 2s
 `
-	psc := processor.MustLoadConfigs(content)
-	obj, err := NewFactory(psc[0].Config, nil)
+	mainConf := processor.MustLoadConfigs(content)[0].Config
+
+	customContent := `
+processor:
+  - name: "db_filter/common"
+    config:
+      slow_query:
+        destination: "db.is_slow"
+        rules:
+        - match: "mysql"
+          threshold: 1s
+`
+	customConf := processor.MustLoadConfigs(customContent)[0].Config
+
+	obj, err := NewFactory(mainConf, []processor.SubConfigProcessor{
+		{
+			Token: "token1",
+			Type:  define.SubConfigFieldDefault,
+			Config: processor.Config{
+				Config: customConf,
+			},
+		},
+	})
 	factory := obj.(*dbFilter)
 	assert.NoError(t, err)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	assert.Equal(t, mainConf, factory.MainConfig())
 
-	c := &Config{}
-	err = mapstructure.Decode(psc[0].Config, c)
-	c.Setup()
+	mainConfig := factory.configs.GetGlobal().(Config)
+	assert.Len(t, mainConfig.SlowQuery.Rules, 2)
 
-	assert.NoError(t, err)
-	assert.Equal(t, *c, factory.configs.GetGlobal().(Config))
+	customConfig := factory.configs.GetByToken("token1").(Config)
+	assert.Len(t, customConfig.SlowQuery.Rules, 1)
 
 	assert.Equal(t, define.ProcessorDbFilter, factory.Name())
 	assert.False(t, factory.IsDerived())
 	assert.False(t, factory.IsPreCheck())
 
-	duration, ok := c.GetSlowQueryConf("mysql")
-	assert.True(t, ok)
-	assert.Equal(t, time.Second, duration)
-
-	factory.Reload(psc[0].Config, nil)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	factory.Reload(mainConf, nil)
+	assert.Equal(t, mainConf, factory.MainConfig())
 }
 
 func TestSlowMySqlQuery(t *testing.T) {

pkg/collector/processor/licensechecker/config.go

@@ -9,7 +9,9 @@
 
 package licensechecker
 
-import "time"
+import (
+	"time"
+)
 
 type Config struct {
 	Enabled           bool          `config:"enabled" mapstructure:"enabled"`

pkg/collector/processor/licensechecker/factory_test.go

@@ -33,24 +33,47 @@ processor:
         number_nodes: 200
         tolerable_num_ratio: 1.5
 `
+	mainConf := processor.MustLoadConfigs(content)[0].Config
 
-	psc := processor.MustLoadConfigs(content)
-	obj, err := NewFactory(psc[0].Config, nil)
+	customContent := `
+processor:
+    - name: "license_checker/common"
+      config:
+        enabled: true
+        expire_time: 110001110
+        tolerable_expire: 24h
+        number_nodes: 200
+        tolerable_num_ratio: 1.5
+`
+	customConf := processor.MustLoadConfigs(customContent)[0].Config
+
+	obj, err := NewFactory(mainConf, []processor.SubConfigProcessor{
+		{
+			Token: "token1",
+			Type:  define.SubConfigFieldDefault,
+			Config: processor.Config{
+				Config: customConf,
+			},
+		},
+	})
 	factory := obj.(*licenseChecker)
 	assert.NoError(t, err)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	assert.Equal(t, mainConf, factory.MainConfig())
 
-	var c Config
-	err = mapstructure.Decode(psc[0].Config, &c)
-	assert.NoError(t, err)
-	assert.Equal(t, c, factory.configs.GetGlobal().(Config))
+	var c1 Config
+	assert.NoError(t, mapstructure.Decode(mainConf, &c1))
+	assert.Equal(t, c1, factory.configs.GetGlobal().(Config))
+
+	var c2 Config
+	assert.NoError(t, mapstructure.Decode(customConf, &c2))
+	assert.Equal(t, c2, factory.configs.GetByToken("token1").(Config))
 
 	assert.Equal(t, define.ProcessorLicenseChecker, factory.Name())
 	assert.False(t, factory.IsDerived())
 	assert.True(t, factory.IsPreCheck())
 
-	factory.Reload(psc[0].Config, nil)
-	assert.Equal(t, psc[0].Config, factory.MainConfig())
+	factory.Reload(mainConf, nil)
+	assert.Equal(t, mainConf, factory.MainConfig())
 }
 
 func TestLicenseCheckerProcess(t *testing.T) {