diff --git a/_sidebar.md b/_sidebar.md index 32d02f0ba..0c18e1ae4 100644 --- a/_sidebar.md +++ b/_sidebar.md @@ -30,6 +30,7 @@ - [flatten](plugin/action/flatten/README.md) - [join](plugin/action/join/README.md) - [json_decode](plugin/action/json_decode/README.md) + - [json_encode](plugin/action/json_encode/README.md) - [keep_fields](plugin/action/keep_fields/README.md) - [mask](plugin/action/mask/README.md) - [modify](plugin/action/modify/README.md) diff --git a/cmd/file.d.go b/cmd/file.d.go index e34fe26ec..7116c467b 100644 --- a/cmd/file.d.go +++ b/cmd/file.d.go @@ -26,6 +26,7 @@ import ( _ "github.com/ozontech/file.d/plugin/action/flatten" _ "github.com/ozontech/file.d/plugin/action/join" _ "github.com/ozontech/file.d/plugin/action/json_decode" + _ "github.com/ozontech/file.d/plugin/action/json_encode" _ "github.com/ozontech/file.d/plugin/action/keep_fields" _ "github.com/ozontech/file.d/plugin/action/mask" _ "github.com/ozontech/file.d/plugin/action/modify" diff --git a/plugin/README.md b/plugin/README.md index 4a957f8ae..bffb19c93 100755 --- a/plugin/README.md +++ b/plugin/README.md @@ -220,6 +220,23 @@ It decodes a JSON string from the event field and merges the result with the eve If the decoded JSON isn't an object, the event will be skipped. [More details...](plugin/action/json_decode/README.md) +## json_encode +It replaces field with its JSON string representation. + +**Example:** +```yaml +pipelines: + example_pipeline: + ... + actions: + - type: json_encode + field: server + ... +``` +It transforms `{"server":{"os":"linux","arch":"amd64"}}` into `{"server":"{\"os\":\"linux\",\"arch\":\"amd64\"}"}`. + + +[More details...](plugin/action/json_encode/README.md) ## keep_fields It keeps the list of the event fields and removes others. diff --git a/plugin/action/README.md b/plugin/action/README.md index c95fcc125..ba76d9599 100755 --- a/plugin/action/README.md +++ b/plugin/action/README.md @@ -78,6 +78,23 @@ It decodes a JSON string from the event field and merges the result with the eve If the decoded JSON isn't an object, the event will be skipped. [More details...](plugin/action/json_decode/README.md) +## json_encode +It replaces field with its JSON string representation. + +**Example:** +```yaml +pipelines: + example_pipeline: + ... + actions: + - type: json_encode + field: server + ... +``` +It transforms `{"server":{"os":"linux","arch":"amd64"}}` into `{"server":"{\"os\":\"linux\",\"arch\":\"amd64\"}"}`. + + +[More details...](plugin/action/json_encode/README.md) ## keep_fields It keeps the list of the event fields and removes others. diff --git a/plugin/action/json_encode/README.idoc.md b/plugin/action/json_encode/README.idoc.md new file mode 100644 index 000000000..7fa526ef2 --- /dev/null +++ b/plugin/action/json_encode/README.idoc.md @@ -0,0 +1,5 @@ +# JSON encode plugin +@introduction + +### Config params +@config-params|description diff --git a/plugin/action/json_encode/README.md b/plugin/action/json_encode/README.md new file mode 100755 index 000000000..9bf4159dc --- /dev/null +++ b/plugin/action/json_encode/README.md @@ -0,0 +1,25 @@ +# JSON encode plugin +It replaces field with its JSON string representation. + +**Example:** +```yaml +pipelines: + example_pipeline: + ... + actions: + - type: json_encode + field: server + ... +``` +It transforms `{"server":{"os":"linux","arch":"amd64"}}` into `{"server":"{\"os\":\"linux\",\"arch\":\"amd64\"}"}`. + + +### Config params +**`field`** *`cfg.FieldSelector`* *`required`* + +The event field to encode. Must be a string. + +
+ + +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/plugin/action/json_encode/json_encode.go b/plugin/action/json_encode/json_encode.go new file mode 100644 index 000000000..f4bcc4b83 --- /dev/null +++ b/plugin/action/json_encode/json_encode.go @@ -0,0 +1,68 @@ +package json_encode + +import ( + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/fd" + "github.com/ozontech/file.d/pipeline" +) + +/*{ introduction +It replaces field with its JSON string representation. + +**Example:** +```yaml +pipelines: + example_pipeline: + ... + actions: + - type: json_encode + field: server + ... +``` +It transforms `{"server":{"os":"linux","arch":"amd64"}}` into `{"server":"{\"os\":\"linux\",\"arch\":\"amd64\"}"}`. + +}*/ +type Plugin struct { + config *Config +} + +//! config-params +//^ config-params +type Config struct { + //> @3@4@5@6 + //> + //> The event field to encode. Must be a string. + Field cfg.FieldSelector `json:"field" parse:"selector" required:"true"` //* + Field_ []string +} + +func init() { + fd.DefaultPluginRegistry.RegisterAction(&pipeline.PluginStaticInfo{ + Type: "json_encode", + Factory: factory, + }) +} + +func factory() (pipeline.AnyPlugin, pipeline.AnyConfig) { + return &Plugin{}, &Config{} +} + +func (p *Plugin) Start(config pipeline.AnyConfig, _ *pipeline.ActionPluginParams) { + p.config = config.(*Config) +} + +func (p *Plugin) Stop() { +} + +func (p *Plugin) Do(event *pipeline.Event) pipeline.ActionResult { + node := event.Root.Dig(p.config.Field_...) + if node == nil { + return pipeline.ActionPass + } + + s := len(event.Buf) + event.Buf = node.Encode(event.Buf) + + node.MutateToString(pipeline.ByteToStringUnsafe(event.Buf[s:])) + return pipeline.ActionPass +} diff --git a/plugin/action/json_encode/json_encode_test.go b/plugin/action/json_encode/json_encode_test.go new file mode 100644 index 000000000..49db9856b --- /dev/null +++ b/plugin/action/json_encode/json_encode_test.go @@ -0,0 +1,44 @@ +package json_encode + +import ( + "sync" + "testing" + + "github.com/ozontech/file.d/cfg" + "github.com/ozontech/file.d/logger" + "github.com/ozontech/file.d/pipeline" + "github.com/ozontech/file.d/test" + "github.com/stretchr/testify/assert" +) + +func TestEncode(t *testing.T) { + config := &Config{Field: "server"} + p, input, output := test.NewPipelineMock(test.NewActionPluginStaticInfo(factory, config, pipeline.MatchModeAnd, nil, false)) + wg := &sync.WaitGroup{} + wg.Add(1) + + err := cfg.Parse(config, nil) + if err != nil { + logger.Panicf("wrong config") + } + + inEvents := 0 + input.SetInFn(func() { + inEvents++ + }) + + outEvents := make([]*pipeline.Event, 0) + output.SetOutFn(func(e *pipeline.Event) { + outEvents = append(outEvents, e) + wg.Done() + }) + + input.In(0, "test.log", 0, []byte(`{"server":{"os":"linux","arch":"amd64"}}`)) + + wg.Wait() + p.Stop() + + assert.Equal(t, 1, inEvents, "wrong in events count") + assert.Equal(t, 1, len(outEvents), "wrong out events count") + assert.Equal(t, `{"server":"{\"os\":\"linux\",\"arch\":\"amd64\"}"}`, outEvents[0].Root.EncodeToString(), "wrong out event") +}