From a79386e1221ff26c26834b20f1b6daa5226652e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Thu, 16 May 2024 18:24:24 +0200 Subject: [PATCH 1/5] Error processor (#1598) * implement error processor * regenerate error processor specs * update conduit-processor-sdk * fix specs * update doc --- go.mod | 2 +- go.sum | 4 +- .../builtin/impl/avro/decode_test.go | 3 +- .../builtin/impl/avro/encode_test.go | 3 +- pkg/plugin/processor/builtin/impl/error.go | 101 ++++++++++++++++++ .../builtin/impl/error_examples_test.go | 47 ++++++++ .../processor/builtin/impl/error_paramgen.go | 19 ++++ .../processor/builtin/impl/error_test.go | 101 ++++++++++++++++++ .../builtin/impl/json/encode_test.go | 5 +- .../builtin/impl/unwrap/opencdc_test.go | 3 +- .../internal/exampleutil/specs/error.json | 45 ++++++++ pkg/plugin/processor/builtin/registry.go | 1 + 12 files changed, 322 insertions(+), 12 deletions(-) create mode 100644 pkg/plugin/processor/builtin/impl/error.go create mode 100644 pkg/plugin/processor/builtin/impl/error_examples_test.go create mode 100644 pkg/plugin/processor/builtin/impl/error_paramgen.go create mode 100644 pkg/plugin/processor/builtin/impl/error_test.go create mode 100644 pkg/plugin/processor/builtin/internal/exampleutil/specs/error.json diff --git a/go.mod b/go.mod index f0929926d..392b204a8 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/conduitio/conduit-connector-protocol v0.6.0 github.com/conduitio/conduit-connector-s3 v0.5.1 github.com/conduitio/conduit-connector-sdk v0.9.1 - github.com/conduitio/conduit-processor-sdk v0.1.1 + github.com/conduitio/conduit-processor-sdk v0.1.2-0.20240516124003-442e4a3f0edd github.com/conduitio/yaml/v3 v3.3.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d diff --git a/go.sum b/go.sum index ebb84e00e..ebd99b6c4 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,8 @@ github.com/conduitio/conduit-connector-s3 v0.5.1 h1:yRo8004ryCIZc/S3iWQ1rN6pm6bj github.com/conduitio/conduit-connector-s3 v0.5.1/go.mod h1:nbxzsyS95gbFJ28Job9vFFB+byRFINSv70/13Yi4mKQ= github.com/conduitio/conduit-connector-sdk v0.9.1 h1:DiMUn7udnjWvyaDsyeTZFHeYTEIdqUU6dqPunEEE3Kw= github.com/conduitio/conduit-connector-sdk v0.9.1/go.mod h1:cNoofumgDlsaThkxkNYg7zab4AkmRZt1V711aO7guGU= -github.com/conduitio/conduit-processor-sdk v0.1.1 h1:C+5Z9pGKVTpdIf5QFNx4UxpvxuOylGRVkGidEpom7HQ= -github.com/conduitio/conduit-processor-sdk v0.1.1/go.mod h1:StkbqQX1WxTjr9LOy7zY+e3DAbEDVvozeamELdzFqck= +github.com/conduitio/conduit-processor-sdk v0.1.2-0.20240516124003-442e4a3f0edd h1:R+tpcZKWOnr6LRsXr85C167SK9MhaLhYUEjBSUupU9Y= +github.com/conduitio/conduit-processor-sdk v0.1.2-0.20240516124003-442e4a3f0edd/go.mod h1:E9zqj0atY1+yBHWi4eZ3TagCZSBnFxBQBUcZktL6RFE= github.com/conduitio/yaml/v3 v3.3.0 h1:kbbaOSHcuH39gP4+rgbJGl6DSbLZcJgEaBvkEXJlCsI= github.com/conduitio/yaml/v3 v3.3.0/go.mod h1:JNgFMOX1t8W4YJuRZOh6GggVtSMsgP9XgTw+7dIenpc= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= diff --git a/pkg/plugin/processor/builtin/impl/avro/decode_test.go b/pkg/plugin/processor/builtin/impl/avro/decode_test.go index 94a0a1acb..faacd39cd 100644 --- a/pkg/plugin/processor/builtin/impl/avro/decode_test.go +++ b/pkg/plugin/processor/builtin/impl/avro/decode_test.go @@ -18,11 +18,10 @@ import ( "context" "testing" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" - "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" "github.com/google/go-cmp/cmp" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/plugin/processor/builtin/impl/avro/encode_test.go b/pkg/plugin/processor/builtin/impl/avro/encode_test.go index ece621292..2a6b06701 100644 --- a/pkg/plugin/processor/builtin/impl/avro/encode_test.go +++ b/pkg/plugin/processor/builtin/impl/avro/encode_test.go @@ -18,11 +18,10 @@ import ( "context" "testing" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" - "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" "github.com/google/go-cmp/cmp" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/plugin/processor/builtin/impl/error.go b/pkg/plugin/processor/builtin/impl/error.go new file mode 100644 index 000000000..1aefd234a --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/error.go @@ -0,0 +1,101 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate paramgen -output=error_paramgen.go errorConfig + +package impl + +import ( + "context" + "strings" + "text/template" + + "github.com/Masterminds/sprig/v3" + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" +) + +type errorConfig struct { + // Error message to be returned. This can be a Go [template](https://pkg.go.dev/text/template) + // executed on each [`Record`](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record) + // being processed. + Message string `json:"message" default:"error processor triggered"` +} + +type errorProcessor struct { + sdk.UnimplementedProcessor + + config errorConfig + errorMessageTmpl *template.Template +} + +func NewErrorProcessor(log.CtxLogger) sdk.Processor { + return &errorProcessor{} +} + +func (p *errorProcessor) Specification() (sdk.Specification, error) { + return sdk.Specification{ + Name: "error", + Summary: "Returns an error for all records that get passed to the processor.", + Description: `Any time a record is passed to this processor it returns an error, +which results in the record being sent to the DLQ if it's configured, or the pipeline stopping. + +**Important:** Make sure to add a [condition](https://conduit.io/docs/processors/conditions) +to this processor, otherwise all records will trigger an error.`, + Version: "v0.1.0", + Author: "Meroxa, Inc.", + Parameters: p.config.Parameters(), + }, nil +} + +func (p *errorProcessor) Configure(ctx context.Context, cfg map[string]string) error { + err := sdk.ParseConfig(ctx, cfg, &p.config, p.config.Parameters()) + if err != nil { + return cerrors.Errorf("failed parsing configuration: %w", err) + } + + if strings.Contains(p.config.Message, "{{") { + // create URL template + p.errorMessageTmpl, err = template.New("").Funcs(sprig.FuncMap()).Parse(p.config.Message) + if err != nil { + return cerrors.Errorf("error while parsing the error message template: %w", err) + } + } + + return nil +} + +func (p *errorProcessor) Process(_ context.Context, records []opencdc.Record) []sdk.ProcessedRecord { + out := make([]sdk.ProcessedRecord, len(records)) + for i := range records { + out[i] = sdk.ErrorRecord{ + Error: cerrors.New(p.errorMessage(records[i])), + } + } + return out +} + +func (p *errorProcessor) errorMessage(record opencdc.Record) string { + if p.errorMessageTmpl == nil { + return p.config.Message + } + + var buf strings.Builder + if err := p.errorMessageTmpl.Execute(&buf, record); err != nil { + return err.Error() + } + return buf.String() +} diff --git a/pkg/plugin/processor/builtin/impl/error_examples_test.go b/pkg/plugin/processor/builtin/impl/error_examples_test.go new file mode 100644 index 000000000..444f69842 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/error_examples_test.go @@ -0,0 +1,47 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package impl + +import ( + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/cerrors" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal/exampleutil" +) + +//nolint:govet // we're using a more descriptive name of example +func ExampleErrorProcessor() { + p := NewErrorProcessor(log.Nop()) + + exampleutil.RunExample(p, exampleutil.Example{ + Summary: `Error record with custom error message`, + Description: `This example shows how to configure the error processor to +return a custom error message for a record using a Go template.`, + Config: map[string]string{ + "message": "custom error message with data from record: {{.Metadata.foo}}", + }, + Have: opencdc.Record{ + Operation: opencdc.OperationCreate, + Metadata: map[string]string{"foo": "bar"}, + Payload: opencdc.Change{After: opencdc.StructuredData{"foo": "bar"}, Before: opencdc.StructuredData{"bar": "baz"}}, + }, + Want: sdk.ErrorRecord{ + Error: cerrors.New("custom error message with data from record: bar"), + }}) + + // Output: + // processor returned error: custom error message with data from record: bar +} diff --git a/pkg/plugin/processor/builtin/impl/error_paramgen.go b/pkg/plugin/processor/builtin/impl/error_paramgen.go new file mode 100644 index 000000000..9e4351892 --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/error_paramgen.go @@ -0,0 +1,19 @@ +// Code generated by paramgen. DO NOT EDIT. +// Source: github.com/ConduitIO/conduit-commons/tree/main/paramgen + +package impl + +import ( + "github.com/conduitio/conduit-commons/config" +) + +func (errorConfig) Parameters() map[string]config.Parameter { + return map[string]config.Parameter{ + "message": { + Default: "error processor triggered", + Description: "Error message to be returned. This can be a Go [template](https://pkg.go.dev/text/template)\nexecuted on each [`Record`](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record)\nbeing processed.", + Type: config.ParameterTypeString, + Validations: []config.Validation{}, + }, + } +} diff --git a/pkg/plugin/processor/builtin/impl/error_test.go b/pkg/plugin/processor/builtin/impl/error_test.go new file mode 100644 index 000000000..5703cef7a --- /dev/null +++ b/pkg/plugin/processor/builtin/impl/error_test.go @@ -0,0 +1,101 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package impl + +import ( + "context" + "testing" + + "github.com/conduitio/conduit-commons/opencdc" + sdk "github.com/conduitio/conduit-processor-sdk" + "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/matryer/is" +) + +func TestError_EmptyConfig(t *testing.T) { + is := is.New(t) + proc := NewErrorProcessor(log.Nop()) + cfg := map[string]string{} + ctx := context.Background() + records := []opencdc.Record{ + { + Metadata: map[string]string{"key1": "val1"}, + Payload: opencdc.Change{ + After: opencdc.StructuredData{ + "foo": "bar", + }, + }, + }, + { + Metadata: map[string]string{"key2": "val2"}, + Payload: opencdc.Change{}, + }, + } + err := proc.Configure(ctx, cfg) + is.NoErr(err) + + got := proc.Process(ctx, records) + is.Equal(len(got), 2) + for _, r := range got { + is.Equal(r.(sdk.ErrorRecord).Error.Error(), "error processor triggered") + } +} + +func TestError_ErrorMessage(t *testing.T) { + records := []opencdc.Record{ + { + Metadata: map[string]string{"foo": "rec 1"}, + Payload: opencdc.Change{ + After: opencdc.StructuredData{ + "foo": "bar", + }, + }, + }, + { + Metadata: map[string]string{"foo": "rec 2"}, + Payload: opencdc.Change{}, + }, + } + testCases := []struct { + name string + cfg map[string]string + wantErrMessages []string + }{{ + name: "static error message", + cfg: map[string]string{"message": "static error message"}, + wantErrMessages: []string{"static error message", "static error message"}, + }, { + name: "template error message", + cfg: map[string]string{"message": "error message: {{.Metadata.foo}}"}, + wantErrMessages: []string{"error message: rec 1", "error message: rec 2"}, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + is := is.New(t) + proc := NewErrorProcessor(log.Nop()) + ctx := context.Background() + + err := proc.Configure(ctx, tc.cfg) + is.NoErr(err) + + got := proc.Process(ctx, records) + is.Equal(len(got), 2) + for i, r := range got { + is.Equal(r.(sdk.ErrorRecord).Error.Error(), tc.wantErrMessages[i]) + } + }) + } +} diff --git a/pkg/plugin/processor/builtin/impl/json/encode_test.go b/pkg/plugin/processor/builtin/impl/json/encode_test.go index 884cfb7cb..4989ff498 100644 --- a/pkg/plugin/processor/builtin/impl/json/encode_test.go +++ b/pkg/plugin/processor/builtin/impl/json/encode_test.go @@ -18,12 +18,11 @@ import ( "context" "testing" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" - "github.com/google/go-cmp/cmp" - "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" + "github.com/google/go-cmp/cmp" "github.com/matryer/is" ) diff --git a/pkg/plugin/processor/builtin/impl/unwrap/opencdc_test.go b/pkg/plugin/processor/builtin/impl/unwrap/opencdc_test.go index c04dd784e..b0b98bb61 100644 --- a/pkg/plugin/processor/builtin/impl/unwrap/opencdc_test.go +++ b/pkg/plugin/processor/builtin/impl/unwrap/opencdc_test.go @@ -18,12 +18,11 @@ import ( "context" "testing" - "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" - "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/plugin/processor/builtin/internal" "github.com/google/go-cmp/cmp" "github.com/matryer/is" ) diff --git a/pkg/plugin/processor/builtin/internal/exampleutil/specs/error.json b/pkg/plugin/processor/builtin/internal/exampleutil/specs/error.json new file mode 100644 index 000000000..179095137 --- /dev/null +++ b/pkg/plugin/processor/builtin/internal/exampleutil/specs/error.json @@ -0,0 +1,45 @@ +{ + "specification": { + "name": "error", + "summary": "Returns an error for all records that get passed to the processor.", + "description": "Any time a record is passed to this processor it returns an error,\nwhich results in the record being sent to the DLQ if it's configured, or the pipeline stopping.\n\n**Important:** Make sure to add a [condition](https://conduit.io/docs/processors/conditions)\nto this processor, otherwise all records will trigger an error.", + "version": "v0.1.0", + "author": "Meroxa, Inc.", + "parameters": { + "message": { + "default": "error processor triggered", + "description": "Error message to be returned. This can be a Go [template](https://pkg.go.dev/text/template)\nexecuted on each [`Record`](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record)\nbeing processed.", + "type": "string", + "validations": [] + } + } + }, + "examples": [ + { + "summary": "Error record with custom error message", + "description": "This example shows how to configure the error processor to\nreturn a custom error message for a record using a Go template.", + "config": { + "message": "custom error message with data from record: {{.Metadata.foo}}" + }, + "have": { + "position": null, + "operation": "create", + "metadata": { + "foo": "bar" + }, + "key": null, + "payload": { + "before": { + "bar": "baz" + }, + "after": { + "foo": "bar" + } + } + }, + "want": { + "error": "custom error message with data from record: bar" + } + } + ] +} diff --git a/pkg/plugin/processor/builtin/registry.go b/pkg/plugin/processor/builtin/registry.go index 03951fd43..6e4024b87 100644 --- a/pkg/plugin/processor/builtin/registry.go +++ b/pkg/plugin/processor/builtin/registry.go @@ -39,6 +39,7 @@ var DefaultBuiltinProcessors = map[string]ProcessorPluginConstructor{ "base64.decode": base64.NewDecodeProcessor, "base64.encode": base64.NewEncodeProcessor, "custom.javascript": custom.NewJavascriptProcessor, + "error": impl.NewErrorProcessor, "filter": impl.NewFilterProcessor, "field.convert": field.NewConvertProcessor, "field.exclude": field.NewExcludeProcessor, From 4641fb48980393f64b7ec965383f4945acde127e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Thu, 16 May 2024 18:41:13 +0200 Subject: [PATCH 2/5] Webhook processor fixes (#1597) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove default for deprecated field, add default content type manually, update example * allow http status <500 * update link to opencdc record * allow go template in request.body --------- Co-authored-by: Haris Osmanagić --- .../processor/builtin/impl/webhook/http.go | 133 ++++++----- .../builtin/impl/webhook/http_config_test.go | 9 +- .../impl/webhook/http_examples_test.go | 15 +- .../builtin/impl/webhook/http_paramgen.go | 6 +- .../builtin/impl/webhook/http_test.go | 216 ++++++++---------- .../exampleutil/specs/webhook.http.json | 15 +- 6 files changed, 191 insertions(+), 203 deletions(-) diff --git a/pkg/plugin/processor/builtin/impl/webhook/http.go b/pkg/plugin/processor/builtin/impl/webhook/http.go index a31635c22..cf02391e7 100644 --- a/pkg/plugin/processor/builtin/impl/webhook/http.go +++ b/pkg/plugin/processor/builtin/impl/webhook/http.go @@ -32,20 +32,19 @@ import ( sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/goccy/go-json" "github.com/jpillora/backoff" ) type httpConfig struct { // URL is a Go template expression for the URL used in the HTTP request, using Go [templates](https://pkg.go.dev/text/template). - // The value provided to the template is [opencdc.Record](https://github.com/ConduitIO/conduit-commons/blob/59ecfbe5d5be2ac4cd9a674d274862d164123f36/opencdc/record.go#L30), - // so the template has access to all its fields (e.g. .Position, .Key, .Metadata, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/) + // The value provided to the template is [opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record), + // so the template has access to all its fields (e.g. `.Position`, `.Key`, `.Metadata`, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/) // to make it easier to write templates. URL string `json:"request.url" validate:"required"` // Method is the HTTP request method to be used. Method string `json:"request.method" default:"GET"` // Deprecated: use `headers.Content-Type` instead. - ContentType string `json:"request.contentType" default:"application/json"` + ContentType string `json:"request.contentType"` // Headers to add to the request, use `headers.*` to specify the header and its value (e.g. `headers.Authorization: "Bearer key"`). Headers map[string]string `json:"headers"` @@ -58,11 +57,13 @@ type httpConfig struct { // The maximum waiting time before retrying. BackoffRetryMax time.Duration `json:"backoffRetry.max" default:"5s"` - // Specifies which field from the input record should be used as the body in - // the HTTP request. + // Specifies the body that will be sent in the HTTP request. The field accepts + // a Go [templates](https://pkg.go.dev/text/template) that's evaluated using the + // [opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record) + // as input. By default, the body is empty. // - // For more information about the format, see [Referencing fields](https://conduit.io/docs/processors/referencing-fields). - RequestBodyRef string `json:"request.body"` + // To send the whole record as JSON you can use `{{ toJson . }}`. + RequestBodyTmpl string `json:"request.body"` // Specifies in which field should the response body be saved. // // For more information about the format, see [Referencing fields](https://conduit.io/docs/processors/referencing-fields). @@ -79,20 +80,25 @@ func (c *httpConfig) parseHeaders() error { c.Headers = make(map[string]string) } - if c.ContentType == "" { - return nil // Nothing to replace in headers - } - + var isContentTypeSet bool for name, _ := range c.Headers { if strings.ToLower(name) == "content-type" { - return cerrors.Errorf("Configuration error, cannot provide both \"request.contentType\" and \"headers.Content-Type\", use \"headers.Content-Type\" only.") + isContentTypeSet = true + break } } - c.Headers["Content-Type"] = c.ContentType - // the ContentType field is deprecated, - // so we're preparing for completely removing it in a later release - c.ContentType = "" + switch { + case isContentTypeSet && c.ContentType != "": + return cerrors.Errorf(`configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only`) + case !isContentTypeSet && c.ContentType != "": + // Use contents of deprecated field. + c.Headers["Content-Type"] = c.ContentType + c.ContentType = "" + case !isContentTypeSet: + // By default, we set the Content-Type to application/json. + c.Headers["Content-Type"] = "application/json" + } return nil } @@ -105,10 +111,11 @@ type httpProcessor struct { config httpConfig backoffCfg *backoff.Backoff - requestBodyRef *sdk.ReferenceResolver + urlTmpl *template.Template + requestBodyTmpl *template.Template + responseBodyRef *sdk.ReferenceResolver responseStatusRef *sdk.ReferenceResolver - urlTmpl *template.Template } func NewHTTPProcessor(l log.CtxLogger) sdk.Processor { @@ -120,7 +127,10 @@ func (p *httpProcessor) Specification() (sdk.Specification, error) { Name: "webhook.http", Summary: "Trigger an HTTP request for every record.", Description: `A processor that sends an HTTP request to the specified URL, retries on error and -saves the response body and, optionally, the response status.`, +saves the response body and, optionally, the response status. + +A status code over 500 is regarded as an error and will cause the processor to retry the request. +The processor will retry the request according to the backoff configuration.`, Version: "v0.1.0", Author: "Meroxa, Inc.", Parameters: httpConfig{}.Parameters(), @@ -142,12 +152,13 @@ func (p *httpProcessor) Configure(ctx context.Context, m map[string]string) erro return cerrors.New("invalid configuration: response.body and response.status set to same field") } - if p.config.RequestBodyRef != "" { - requestBodyRef, err := sdk.NewReferenceResolver(p.config.RequestBodyRef) + // parse request body template + if strings.Contains(p.config.RequestBodyTmpl, "{{") { + // create URL template + p.requestBodyTmpl, err = template.New("").Funcs(sprig.FuncMap()).Parse(p.config.RequestBodyTmpl) if err != nil { - return cerrors.Errorf("failed parsing request.body %v: %w", p.config.RequestBodyRef, err) + return cerrors.Errorf("failed parsing request.body %v: %w", err) } - p.requestBodyRef = &requestBodyRef } responseBodyRef, err := sdk.NewReferenceResolver(p.config.ResponseBodyRef) @@ -164,6 +175,8 @@ func (p *httpProcessor) Configure(ctx context.Context, m map[string]string) erro } p.responseStatusRef = &responseStatusRef } + + // parse URL template if strings.Contains(p.config.URL, "{{") { // create URL template p.urlTmpl, err = template.New("").Funcs(sprig.FuncMap()).Parse(p.config.URL) @@ -186,27 +199,6 @@ func (p *httpProcessor) Configure(ctx context.Context, m map[string]string) erro return nil } -func (p *httpProcessor) EvaluateURL(rec opencdc.Record) (string, error) { - if p.urlTmpl == nil { - return p.config.URL, nil - } - var b bytes.Buffer - err := p.urlTmpl.Execute(&b, rec) - if err != nil { - return "", cerrors.Errorf("error while evaluating URL template: %w", err) - } - u, err := url.Parse(b.String()) - if err != nil { - return "", cerrors.Errorf("error parsing URL: %w", err) - } - q, err := url.ParseQuery(u.RawQuery) - if err != nil { - return "", cerrors.Errorf("error parsing URL query: %w", err) - } - u.RawQuery = q.Encode() - return u.String(), nil -} - func (p *httpProcessor) Process(ctx context.Context, records []opencdc.Record) []sdk.ProcessedRecord { out := make([]sdk.ProcessedRecord, 0, len(records)) for _, rec := range records { @@ -281,14 +273,10 @@ func (p *httpProcessor) processRecord(ctx context.Context, r opencdc.Record) (sd return nil, cerrors.Errorf("error reading response body: %w", err) } - if resp.StatusCode >= 300 { - // regard status codes over 299 as errors + if resp.StatusCode >= 500 { + // regard status codes over 500 as errors return nil, cerrors.Errorf("error status code %v (body: %q)", resp.StatusCode, string(body)) } - // skip if body has no content - if resp.StatusCode == http.StatusNoContent { - return sdk.FilterRecord{}, nil - } // Set response body err = p.setField(&r, p.responseBodyRef, body) @@ -309,7 +297,7 @@ func (p *httpProcessor) buildRequest(ctx context.Context, r opencdc.Record) (*ht return nil, cerrors.Errorf("failed getting request body: %w", err) } - url, err := p.EvaluateURL(r) + url, err := p.evaluateURL(r) if err != nil { return nil, err } @@ -331,25 +319,44 @@ func (p *httpProcessor) buildRequest(ctx context.Context, r opencdc.Record) (*ht return req, nil } +func (p *httpProcessor) evaluateURL(rec opencdc.Record) (string, error) { + if p.urlTmpl == nil { + return p.config.URL, nil + } + var b bytes.Buffer + err := p.urlTmpl.Execute(&b, rec) + if err != nil { + return "", cerrors.Errorf("error while evaluating URL template: %w", err) + } + u, err := url.Parse(b.String()) + if err != nil { + return "", cerrors.Errorf("error parsing URL: %w", err) + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", cerrors.Errorf("error parsing URL query: %w", err) + } + u.RawQuery = q.Encode() + return u.String(), nil +} + // requestBody returns the request body for the given record, // using the configured field reference (see: request.body configuration parameter). func (p *httpProcessor) requestBody(r opencdc.Record) ([]byte, error) { - if p.requestBodyRef == nil { + if p.requestBodyTmpl == nil { + if p.config.RequestBodyTmpl != "" { + return []byte(p.config.RequestBodyTmpl), nil + } return nil, nil } - ref, err := p.requestBodyRef.Resolve(&r) - if err != nil { - return nil, cerrors.Errorf("failed resolving request.body: %w", err) - } - val := ref.Get() - // Raw byte data should be sent as it is, as that's most often what we want - // If we json.Marshal it first, it will be Base64-encoded. - if raw, ok := val.(opencdc.RawData); ok { - return raw.Bytes(), nil + var b bytes.Buffer + err := p.requestBodyTmpl.Execute(&b, r) + if err != nil { + return nil, cerrors.Errorf("error while evaluating request body template: %w", err) } - return json.Marshal(val) + return b.Bytes(), nil } func (p *httpProcessor) setField(r *opencdc.Record, refRes *sdk.ReferenceResolver, data any) error { diff --git a/pkg/plugin/processor/builtin/impl/webhook/http_config_test.go b/pkg/plugin/processor/builtin/impl/webhook/http_config_test.go index c13f5dbf5..89cd3baba 100644 --- a/pkg/plugin/processor/builtin/impl/webhook/http_config_test.go +++ b/pkg/plugin/processor/builtin/impl/webhook/http_config_test.go @@ -15,11 +15,12 @@ package webhook import ( - "github.com/matryer/is" "testing" + + "github.com/matryer/is" ) -func TestHTTPConfig_ValidateHeaders(t *testing.T) { +func TestHTTPConfig_ParseHeaders(t *testing.T) { testCases := []struct { name string input httpConfig @@ -34,7 +35,7 @@ func TestHTTPConfig_ValidateHeaders(t *testing.T) { "Content-Type": "application/json", }, }, - wantErr: `Configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only.`, + wantErr: `configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only`, }, { name: "ContentType field present, header present, different case", @@ -44,7 +45,7 @@ func TestHTTPConfig_ValidateHeaders(t *testing.T) { "content-type": "application/json", }, }, - wantErr: `Configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only.`, + wantErr: `configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only`, }, { name: "ContentType field presents, header not present", diff --git a/pkg/plugin/processor/builtin/impl/webhook/http_examples_test.go b/pkg/plugin/processor/builtin/impl/webhook/http_examples_test.go index 2df0c469a..6c6c36548 100644 --- a/pkg/plugin/processor/builtin/impl/webhook/http_examples_test.go +++ b/pkg/plugin/processor/builtin/impl/webhook/http_examples_test.go @@ -39,15 +39,16 @@ func ExampleHTTPProcessor() { exampleutil.RunExample(p, exampleutil.Example{ Summary: `Send a request to an HTTP server`, Description: ` -This example shows how to use the HTTP processor to send a record's ` + "`.Payload.After`" + ` field to a dummy HTTP server -that replies back with a greeting. +This example shows how to use the HTTP processor to send a record's ` + "`.Payload.After`" + ` field as a string to a dummy +HTTP server that replies back with a greeting. -The record's ` + "`.Payload.After`" + ` is overwritten with the response. Additionally, the example shows how to store the -value of the HTTP response's code in the metadata field ` + "`http_status`" + `.`, +The record's ` + "`.Payload.After`" + ` is overwritten with the response. Additionally, the example shows how to set a request +header and how to store the value of the HTTP response's code in the metadata field ` + "`http_status`" + `.`, Config: map[string]string{ - "request.url": srv.URL, - "request.body": ".Payload.After", - "response.status": `.Metadata["http_status"]`, + "request.url": srv.URL, + "request.body": `{{ printf "%s" .Payload.After }}`, + "response.status": `.Metadata["http_status"]`, + "headers.content-type": "application/json", }, Have: opencdc.Record{ Operation: opencdc.OperationUpdate, diff --git a/pkg/plugin/processor/builtin/impl/webhook/http_paramgen.go b/pkg/plugin/processor/builtin/impl/webhook/http_paramgen.go index fc86fe27e..b695cc430 100644 --- a/pkg/plugin/processor/builtin/impl/webhook/http_paramgen.go +++ b/pkg/plugin/processor/builtin/impl/webhook/http_paramgen.go @@ -45,12 +45,12 @@ func (httpConfig) Parameters() map[string]config.Parameter { }, "request.body": { Default: "", - Description: "Specifies which field from the input record should be used as the body in\nthe HTTP request.\n\nFor more information about the format, see [Referencing fields](https://conduit.io/docs/processors/referencing-fields).", + Description: "Specifies the body that will be sent in the HTTP request. The field accepts\na Go [templates](https://pkg.go.dev/text/template) that's evaluated using the\n[opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record)\nas input. By default, the body is empty.\n\nTo send the whole record as JSON you can use `{{ toJson . }}`.", Type: config.ParameterTypeString, Validations: []config.Validation{}, }, "request.contentType": { - Default: "application/json", + Default: "", Description: "Deprecated: use `headers.Content-Type` instead.", Type: config.ParameterTypeString, Validations: []config.Validation{}, @@ -63,7 +63,7 @@ func (httpConfig) Parameters() map[string]config.Parameter { }, "request.url": { Default: "", - Description: "URL is a Go template expression for the URL used in the HTTP request, using Go [templates](https://pkg.go.dev/text/template).\nThe value provided to the template is [opencdc.Record](https://github.com/ConduitIO/conduit-commons/blob/59ecfbe5d5be2ac4cd9a674d274862d164123f36/opencdc/record.go#L30),\nso the template has access to all its fields (e.g. .Position, .Key, .Metadata, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/)\nto make it easier to write templates.", + Description: "URL is a Go template expression for the URL used in the HTTP request, using Go [templates](https://pkg.go.dev/text/template).\nThe value provided to the template is [opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record),\nso the template has access to all its fields (e.g. `.Position`, `.Key`, `.Metadata`, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/)\nto make it easier to write templates.", Type: config.ParameterTypeString, Validations: []config.Validation{ config.ValidationRequired{}, diff --git a/pkg/plugin/processor/builtin/impl/webhook/http_test.go b/pkg/plugin/processor/builtin/impl/webhook/http_test.go index 7ca3d7e48..ac551453d 100644 --- a/pkg/plugin/processor/builtin/impl/webhook/http_test.go +++ b/pkg/plugin/processor/builtin/impl/webhook/http_test.go @@ -24,10 +24,8 @@ import ( "github.com/conduitio/conduit-commons/opencdc" sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/goccy/go-json" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/matryer/is" ) @@ -140,6 +138,23 @@ func TestHTTPProcessor_Configure(t *testing.T) { }, wantErr: "", }, + { + name: "content-type header", + config: map[string]string{ + "request.url": "http://example.com", + "headers.content-type": "application/json", + }, + wantErr: "", + }, + { + name: "invalid: content-type header and request.contentType", + config: map[string]string{ + "request.url": "http://example.com", + "request.contentType": "application/json", + "headers.content-type": "application/json", + }, + wantErr: `configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only`, + }, { name: "invalid: same value of response.body and response.status", config: map[string]string{ @@ -179,18 +194,22 @@ func TestHTTPProcessor_Success(t *testing.T) { respBody := []byte("foo-bar/response") tests := []struct { - name string - config map[string]string - args []opencdc.Record - want []sdk.ProcessedRecord + name string + config map[string]string + status int + record opencdc.Record + wantBody string + want sdk.ProcessedRecord }{ { name: "structured data", config: map[string]string{ "request.method": "POST", - "request.body": ".", + "request.body": "{{ toJson . }}", }, - args: []opencdc.Record{{ + status: 200, + record: opencdc.Record{ + Operation: opencdc.OperationCreate, Payload: opencdc.Change{ Before: nil, After: opencdc.StructuredData{ @@ -198,30 +217,35 @@ func TestHTTPProcessor_Success(t *testing.T) { "baz": nil, }, }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `{"position":null,"operation":"create","metadata":null,"key":null,"payload":{"before":null,"after":{"bar":123,"baz":null}}}`, + want: sdk.SingleRecord{ + Operation: opencdc.OperationCreate, Payload: opencdc.Change{ After: opencdc.RawData(respBody), }, }, - }, }, { name: "raw data", config: map[string]string{ "request.method": "GET", - "request.body": ".", + "request.body": "{{ toJson . }}", }, - args: []opencdc.Record{{ + status: 200, + record: opencdc.Record{ + Operation: opencdc.OperationUpdate, Payload: opencdc.Change{ After: opencdc.RawData("random data"), }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `{"position":null,"operation":"update","metadata":null,"key":null,"payload":{"before":null,"after":"cmFuZG9tIGRhdGE="}}`, + want: sdk.SingleRecord{ + Operation: opencdc.OperationUpdate, Payload: opencdc.Change{ After: opencdc.RawData(respBody), }, - }}, + }, }, { name: "custom field for response body and status", @@ -229,33 +253,39 @@ func TestHTTPProcessor_Success(t *testing.T) { "response.body": ".Payload.After.body", "response.status": ".Payload.After.status", "request.method": "POST", - "request.body": ".", + "request.body": "{{ toJson . }}", }, - args: []opencdc.Record{{ + status: 404, + record: opencdc.Record{ + Operation: opencdc.OperationSnapshot, Payload: opencdc.Change{ After: opencdc.StructuredData{ "a key": "random data", }, }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `{"position":null,"operation":"snapshot","metadata":null,"key":null,"payload":{"before":null,"after":{"a key":"random data"}}}`, + want: sdk.SingleRecord{ + Operation: opencdc.OperationSnapshot, Payload: opencdc.Change{ After: opencdc.StructuredData{ "a key": "random data", "body": respBody, - "status": "200", + "status": "404", }, }, - }}, + }, }, { name: "request body: custom field, structured", config: map[string]string{ - "request.body": ".", + "request.body": "{{ toJson . }}", "response.body": ".Payload.After.httpResponse", "request.method": "POST", }, - args: []opencdc.Record{{ + status: 200, + record: opencdc.Record{ + Operation: opencdc.OperationDelete, Payload: opencdc.Change{ Before: opencdc.StructuredData{ "before-key": "before-data", @@ -264,8 +294,10 @@ func TestHTTPProcessor_Success(t *testing.T) { "after-key": "after-data", }, }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `{"position":null,"operation":"delete","metadata":null,"key":null,"payload":{"before":{"before-key":"before-data"},"after":{"after-key":"after-data"}}}`, + want: sdk.SingleRecord{ + Operation: opencdc.OperationDelete, Payload: opencdc.Change{ Before: opencdc.StructuredData{ "before-key": "before-data", @@ -275,24 +307,26 @@ func TestHTTPProcessor_Success(t *testing.T) { "httpResponse": []byte("foo-bar/response"), }, }, - }}, + }, }, { name: "request body: custom field, raw data", config: map[string]string{ - "request.body": ".Payload.Before", + "request.body": `{{ printf "%s" .Payload.Before }}`, "response.body": ".Payload.After.httpResponse", "request.method": "POST", }, - args: []opencdc.Record{{ + status: 200, + record: opencdc.Record{ Payload: opencdc.Change{ Before: opencdc.RawData("uncooked data"), After: opencdc.StructuredData{ "after-key": "after-data", }, }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `uncooked data`, + want: sdk.SingleRecord{ Payload: opencdc.Change{ Before: opencdc.RawData("uncooked data"), After: opencdc.StructuredData{ @@ -300,53 +334,53 @@ func TestHTTPProcessor_Success(t *testing.T) { "httpResponse": []byte("foo-bar/response"), }, }, - }}, + }, }, { - name: "request body: custom field, []byte data", + name: "request body: static", config: map[string]string{ - "request.body": ".Payload.After.contents", + "request.body": `foo`, "response.body": ".Payload.After.httpResponse", "request.method": "POST", }, - args: []opencdc.Record{{ + status: 200, + record: opencdc.Record{ Payload: opencdc.Change{ After: opencdc.StructuredData{ - "contents": []byte{15, 2, 20, 24}, - "after-key": "after-data", + "unused": "data", }, }, - }}, - want: []sdk.ProcessedRecord{sdk.SingleRecord{ + }, + wantBody: `foo`, + want: sdk.SingleRecord{ Payload: opencdc.Change{ After: opencdc.StructuredData{ - "after-key": "after-data", - "contents": []byte{15, 2, 20, 24}, + "unused": "data", "httpResponse": []byte("foo-bar/response"), }, }, - }}, + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { is := is.New(t) + ctx := context.Background() wantMethod := tc.config["request.method"] if wantMethod == "" { wantMethod = "GET" // default } - wantBody := getRequestBody(is, tc.config["request.body"], tc.args) - srv := httptest.NewServer(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { is.Equal(wantMethod, req.Method) gotBody, err := io.ReadAll(req.Body) is.NoErr(err) - is.Equal(wantBody, gotBody) + is.Equal(tc.wantBody, string(gotBody)) + resp.WriteHeader(tc.status) _, err = resp.Write(respBody) is.NoErr(err) })) @@ -354,15 +388,18 @@ func TestHTTPProcessor_Success(t *testing.T) { tc.config["request.url"] = srv.URL underTest := NewHTTPProcessor(log.Test(t)) - err := underTest.Configure(context.Background(), tc.config) + err := underTest.Configure(ctx, tc.config) is.NoErr(err) - got := underTest.Process(context.Background(), tc.args) - diff := cmp.Diff(tc.want, got, cmpopts.IgnoreUnexported(sdk.SingleRecord{})) - if diff != "" { - t.Logf("mismatch (-want +got): %s", diff) - t.Fail() - } + got := underTest.Process(ctx, []opencdc.Record{tc.record}) + is.Equal( + "", + cmp.Diff( + []sdk.ProcessedRecord{tc.want}, + got, + cmpopts.IgnoreUnexported(sdk.SingleRecord{}), + ), + ) }) } } @@ -372,44 +409,44 @@ func TestHTTPProcessor_URLTemplate(t *testing.T) { name string pathTmpl string // will be attached to the URL path string // expected result of the pathTmpl - args []opencdc.Record + record opencdc.Record }{ { name: "URL template, success", pathTmpl: "/{{.Payload.After.foo}}", path: "/123", - args: []opencdc.Record{{ + record: opencdc.Record{ Payload: opencdc.Change{ Before: nil, After: opencdc.StructuredData{ "foo": 123, }, }, - }}, + }, }, { name: "URL template, key with a hyphen", pathTmpl: `/{{index .Payload.After "foo-bar"}}`, path: "/baz", - args: []opencdc.Record{{ + record: opencdc.Record{ Payload: opencdc.Change{ After: opencdc.StructuredData{ "foo-bar": "baz", }, }, - }}, + }, }, { name: "URL template, path and query have spaces", pathTmpl: `/{{.Payload.Before.url}}`, path: "/what%20is%20conduit?id=my+id", - args: []opencdc.Record{{ + record: opencdc.Record{ Payload: opencdc.Change{ Before: opencdc.StructuredData{ "url": "what is conduit?id=my id", }, }, - }}, + }, }, } @@ -431,7 +468,7 @@ func TestHTTPProcessor_URLTemplate(t *testing.T) { err := underTest.Configure(context.Background(), config) is.NoErr(err) - got := underTest.Process(context.Background(), tc.args) + got := underTest.Process(context.Background(), []opencdc.Record{tc.record}) is.Equal(1, len(got)) _, ok := got[0].(sdk.SingleRecord) is.True(ok) @@ -477,7 +514,7 @@ func TestHTTPProcessor_RetrySuccess(t *testing.T) { "backoffRetry.min": "5ms", "backoffRetry.max": "10ms", "backoffRetry.factor": "1.2", - "request.body": ".", + "request.body": "{{ toJson . }}", } underTest := NewHTTPProcessor(log.Test(t)) @@ -529,60 +566,3 @@ func TestHTTPProcessor_RetryFail(t *testing.T) { is.True(isErr) // expected an error is.Equal(srvHandlerCount, 6) // expected 6 requests (1 regular and 5 retries) } - -func TestHTTPProcessor_FilterRecord(t *testing.T) { - is := is.New(t) - - wantMethod := "GET" - rec := []opencdc.Record{ - {Payload: opencdc.Change{After: opencdc.RawData("random data")}}, - } - - wantBody := getRequestBody(is, ".", rec) - - srv := httptest.NewServer(http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - is.Equal(wantMethod, req.Method) - - gotBody, err := io.ReadAll(req.Body) - is.NoErr(err) - is.Equal(wantBody, gotBody) - - resp.WriteHeader(http.StatusNoContent) - })) - defer srv.Close() - - config := map[string]string{ - "request.url": srv.URL, - "request.body": ".", - } - - underTest := NewHTTPProcessor(log.Test(t)) - err := underTest.Configure(context.Background(), config) - is.NoErr(err) - - got := underTest.Process(context.Background(), rec) - is.Equal(got, []sdk.ProcessedRecord{sdk.FilterRecord{}}) -} - -func getRequestBody(is *is.I, field string, records []opencdc.Record) []byte { - f := field - if f == "" { - f = "." - } - - refRes, err := sdk.NewReferenceResolver(f) - is.NoErr(err) - - ref, err := refRes.Resolve(&records[0]) - is.NoErr(err) - - val := ref.Get() - if raw, ok := val.(opencdc.RawData); ok { - return raw.Bytes() - } - - bytes, err := json.Marshal(ref.Get()) - is.NoErr(err) - - return bytes -} diff --git a/pkg/plugin/processor/builtin/internal/exampleutil/specs/webhook.http.json b/pkg/plugin/processor/builtin/internal/exampleutil/specs/webhook.http.json index 54a5e2ac2..0bcf6702e 100644 --- a/pkg/plugin/processor/builtin/internal/exampleutil/specs/webhook.http.json +++ b/pkg/plugin/processor/builtin/internal/exampleutil/specs/webhook.http.json @@ -2,7 +2,7 @@ "specification": { "name": "webhook.http", "summary": "Trigger an HTTP request for every record.", - "description": "A processor that sends an HTTP request to the specified URL, retries on error and \nsaves the response body and, optionally, the response status.", + "description": "A processor that sends an HTTP request to the specified URL, retries on error and \nsaves the response body and, optionally, the response status.\n\nA status code over 500 is regarded as an error and will cause the processor to retry the request.\nThe processor will retry the request according to the backoff configuration.", "version": "v0.1.0", "author": "Meroxa, Inc.", "parameters": { @@ -48,12 +48,12 @@ }, "request.body": { "default": "", - "description": "Specifies which field from the input record should be used as the body in\nthe HTTP request.\n\nFor more information about the format, see [Referencing fields](https://conduit.io/docs/processors/referencing-fields).", + "description": "Specifies the body that will be sent in the HTTP request. The field accepts\na Go [templates](https://pkg.go.dev/text/template) that's evaluated using the\n[opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record)\nas input. By default, the body is empty.\n\nTo send the whole record as JSON you can use `{{ toJson . }}`.", "type": "string", "validations": [] }, "request.contentType": { - "default": "application/json", + "default": "", "description": "Deprecated: use `headers.Content-Type` instead.", "type": "string", "validations": [] @@ -66,7 +66,7 @@ }, "request.url": { "default": "", - "description": "URL is a Go template expression for the URL used in the HTTP request, using Go [templates](https://pkg.go.dev/text/template).\nThe value provided to the template is [opencdc.Record](https://github.com/ConduitIO/conduit-commons/blob/59ecfbe5d5be2ac4cd9a674d274862d164123f36/opencdc/record.go#L30),\nso the template has access to all its fields (e.g. .Position, .Key, .Metadata, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/)\nto make it easier to write templates.", + "description": "URL is a Go template expression for the URL used in the HTTP request, using Go [templates](https://pkg.go.dev/text/template).\nThe value provided to the template is [opencdc.Record](https://pkg.go.dev/github.com/conduitio/conduit-commons/opencdc#Record),\nso the template has access to all its fields (e.g. `.Position`, `.Key`, `.Metadata`, and so on). We also inject all template functions provided by [sprig](https://masterminds.github.io/sprig/)\nto make it easier to write templates.", "type": "string", "validations": [ { @@ -92,14 +92,14 @@ "examples": [ { "summary": "Send a request to an HTTP server", - "description": "\nThis example shows how to use the HTTP processor to send a record's `.Payload.After` field to a dummy HTTP server\nthat replies back with a greeting.\n\nThe record's `.Payload.After` is overwritten with the response. Additionally, the example shows how to store the\nvalue of the HTTP response's code in the metadata field `http_status`.", + "description": "\nThis example shows how to use the HTTP processor to send a record's `.Payload.After` field as a string to a dummy\nHTTP server that replies back with a greeting.\n\nThe record's `.Payload.After` is overwritten with the response. Additionally, the example shows how to set a request\nheader and how to store the value of the HTTP response's code in the metadata field `http_status`.", "config": { "backoffRetry.count": "0", "backoffRetry.factor": "2", "backoffRetry.max": "5s", "backoffRetry.min": "100ms", - "request.body": ".Payload.After", - "request.contentType": "application/json", + "headers.content-type": "application/json", + "request.body": "{{ printf \"%s\" .Payload.After }}", "request.method": "GET", "request.url": "http://127.0.0.1:54321", "response.body": ".Payload.After", @@ -136,7 +136,6 @@ "backoffRetry.factor": "2", "backoffRetry.max": "5s", "backoffRetry.min": "100ms", - "request.contentType": "application/json", "request.method": "GET", "request.url": "http://127.0.0.1:54321/{{.Payload.After.name}}", "response.body": ".Payload.After.response" From 60f65b7fbe166cdbdb7e26c09a6a9d036f518c41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 09:14:24 +0000 Subject: [PATCH 3/5] go.mod: bump github.com/grpc-ecosystem/grpc-gateway/v2 (#1595) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/grpc-ecosystem/grpc-gateway/v2](https://github.com/grpc-ecosystem/grpc-gateway) from 2.19.1 to 2.20.0. - [Release notes](https://github.com/grpc-ecosystem/grpc-gateway/releases) - [Changelog](https://github.com/grpc-ecosystem/grpc-gateway/blob/main/.goreleaser.yml) - [Commits](https://github.com/grpc-ecosystem/grpc-gateway/compare/v2.19.1...v2.20.0) --- updated-dependencies: - dependency-name: github.com/grpc-ecosystem/grpc-gateway/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Raúl Barroso --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 392b204a8..f7a4ade18 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 github.com/hamba/avro/v2 v2.21.1 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.1 @@ -51,7 +51,7 @@ require ( golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f golang.org/x/tools v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 - google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be + google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 @@ -329,7 +329,7 @@ require ( golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index ebd99b6c4..41066639f 100644 --- a/go.sum +++ b/go.sum @@ -677,8 +677,8 @@ github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hamba/avro/v2 v2.21.1 h1:400/jTdLWQ3ib58y83VXlTJKijRouYQszY1SO0cMGt4= github.com/hamba/avro/v2 v2.21.1/go.mod h1:ouJ4PkiAEP49u0lAtQyd5Gv04MehKj+7lXwD3zpLpY0= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= @@ -1773,10 +1773,10 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= -google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= +google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 12b9246af79a875deccaada34ecb3291acb9538d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 09:37:46 +0000 Subject: [PATCH 4/5] go.mod: bump buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go (#1600) --- updated-dependencies: - dependency-name: buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f7a4ade18..8532f5e12 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conduitio/conduit go 1.22.2 require ( - buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1 + buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240516201131-8c5085307891.1 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/NYTimes/gziphandler v1.1.1 diff --git a/go.sum b/go.sum index 41066639f..0d80a3c9f 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ 4d63.com/gochecknoglobals v0.2.1/go.mod h1:KRE8wtJB3CXCsb1xy421JfTHIIbmT3U5ruxw2Qu8fSU= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is= buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw= -buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1 h1:AhgzTziVBgbY6sCwiNmn4qwgQxmYrPt5jggS2JQOPtk= -buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1/go.mod h1:gUfBTdJmhwTJHoIaOQjy7xczHhqjkQfXSKs5TY4IsUY= +buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240516201131-8c5085307891.1 h1:VqxQj/MuTTC/vUOfIR2nX+g8Xy60mJojJXG+mvzvFPA= +buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240516201131-8c5085307891.1/go.mod h1:gUfBTdJmhwTJHoIaOQjy7xczHhqjkQfXSKs5TY4IsUY= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= From 8ca83c2cc84016620e2056240a904d2d85d0e451 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 09:48:17 +0000 Subject: [PATCH 5/5] go.mod: bump github.com/bufbuild/buf from 1.31.0 to 1.32.0 (#1599) Bumps [github.com/bufbuild/buf](https://github.com/bufbuild/buf) from 1.31.0 to 1.32.0. - [Release notes](https://github.com/bufbuild/buf/releases) - [Changelog](https://github.com/bufbuild/buf/blob/main/CHANGELOG.md) - [Commits](https://github.com/bufbuild/buf/compare/v1.31.0...v1.32.0) --- updated-dependencies: - dependency-name: github.com/bufbuild/buf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 25 ++++++++++++++----------- go.sum | 56 +++++++++++++++++++++++++++++++------------------------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 8532f5e12..1a1d86445 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/NYTimes/gziphandler v1.1.1 - github.com/bufbuild/buf v1.31.0 + github.com/bufbuild/buf v1.32.0 github.com/conduitio/conduit-commons v0.2.0 github.com/conduitio/conduit-connector-file v0.6.0 github.com/conduitio/conduit-connector-generator v0.6.0 @@ -48,10 +48,10 @@ require ( github.com/twmb/go-cache v1.2.1 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.4.0 - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/tools v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 - google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 + google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 @@ -60,7 +60,9 @@ require ( require ( 4d63.com/gocheckcompilerdirectives v1.2.1 // indirect 4d63.com/gochecknoglobals v0.2.1 // indirect - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 // indirect + buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.1-20240514010100-299bd9c9a0c4.1 // indirect + buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240514010100-299bd9c9a0c4.1 // indirect connectrpc.com/connect v1.16.1 // indirect connectrpc.com/otelconnect v0.7.0 // indirect github.com/4meepo/tagalign v1.3.4 // indirect @@ -110,7 +112,8 @@ require ( github.com/bombsimon/wsl/v4 v4.2.1 // indirect github.com/breml/bidichk v0.2.7 // indirect github.com/breml/errchkjson v0.3.6 // indirect - github.com/bufbuild/protocompile v0.9.0 // indirect + github.com/bufbuild/protocompile v0.13.1-0.20240510201809-752249dfc37f // indirect + github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee // indirect github.com/bufbuild/protovalidate-go v0.6.2 // indirect github.com/bufbuild/protoyaml-go v0.1.9 // indirect github.com/butuzov/ireturn v0.3.0 // indirect @@ -130,9 +133,9 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/docker/cli v26.1.0+incompatible // indirect + github.com/docker/cli v26.1.2+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v26.1.0+incompatible // indirect + github.com/docker/docker v26.1.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -162,7 +165,7 @@ require ( github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.8.1 // indirect - github.com/gofrs/uuid/v5 v5.1.0 // indirect + github.com/gofrs/uuid/v5 v5.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -178,7 +181,7 @@ require ( github.com/google/cel-go v0.20.1 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/go-containerregistry v0.19.1 // indirect - github.com/google/pprof v0.0.0-20240422182052-72c8669ad3e7 // indirect + github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.4.2 // indirect @@ -260,7 +263,7 @@ require ( github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/rs/cors v1.10.1 // indirect + github.com/rs/cors v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryancurrah/gomodguard v1.3.2 // indirect github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect @@ -329,7 +332,7 @@ require ( golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 0d80a3c9f..4d79994a0 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,13 @@ 4d63.com/gocheckcompilerdirectives v1.2.1/go.mod h1:yjDJSxmDTtIHHCqX0ufRYZDL6vQtMG7tJdKVeWwsqvs= 4d63.com/gochecknoglobals v0.2.1 h1:1eiorGsgHOFOuoOiJDy2psSrQbRdIHrlge0IJIkUgDc= 4d63.com/gochecknoglobals v0.2.1/go.mod h1:KRE8wtJB3CXCsb1xy421JfTHIIbmT3U5ruxw2Qu8fSU= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240401165935-b983156c5e99.1/go.mod h1:XF+P8+RmfdufmIYpGUC+6bF7S+IlmHDEnCrO3OXaUAQ= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1 h1:LEXWFH/xZ5oOWrC3oOtHbUyBdzRWMCPpAQmKC9v05mA= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.34.1-20240508200655-46a4cf4ba109.1/go.mod h1:XF+P8+RmfdufmIYpGUC+6bF7S+IlmHDEnCrO3OXaUAQ= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.1-20240514010100-299bd9c9a0c4.1 h1:6GN3XU5DaLjZb2vdle1za4TCwRHlEXMGSFCmHz4FM3w= +buf.build/gen/go/bufbuild/registry/connectrpc/go v1.16.1-20240514010100-299bd9c9a0c4.1/go.mod h1:3Vwq3HRCNOcnv99Ra+/8K5cntwO/Gw98cIjsjQVxf4E= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240514010100-299bd9c9a0c4.1 h1:zrXXp1IT3qukIKorguvnrw/JzxYttBrXzktUQGiQHqA= +buf.build/gen/go/bufbuild/registry/protocolbuffers/go v1.34.1-20240514010100-299bd9c9a0c4.1/go.mod h1:8ONhsyCTLQ9kBslWnMgPrXTcxzCkKlxZqN9ewUveui8= buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240516201131-8c5085307891.1 h1:VqxQj/MuTTC/vUOfIR2nX+g8Xy60mJojJXG+mvzvFPA= buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240516201131-8c5085307891.1/go.mod h1:gUfBTdJmhwTJHoIaOQjy7xczHhqjkQfXSKs5TY4IsUY= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -266,10 +271,12 @@ github.com/breml/bidichk v0.2.7 h1:dAkKQPLl/Qrk7hnP6P+E0xOodrq8Us7+U0o4UBOAlQY= github.com/breml/bidichk v0.2.7/go.mod h1:YodjipAGI9fGcYM7II6wFvGhdMYsC5pHDlGzqvEW3tQ= github.com/breml/errchkjson v0.3.6 h1:VLhVkqSBH96AvXEyclMR37rZslRrY2kcyq+31HCsVrA= github.com/breml/errchkjson v0.3.6/go.mod h1:jhSDoFheAF2RSDOlCfhHO9KqhZgAYLyvHe7bRCX8f/U= -github.com/bufbuild/buf v1.31.0 h1:YHLGIr8bjcLaTCIw0+/bCAvJLiR8u46QTwKvn7miSEg= -github.com/bufbuild/buf v1.31.0/go.mod h1:LlxpG2LF33f1Ixw29BTt0pyLriLzg3rXY1K9XQVHSio= -github.com/bufbuild/protocompile v0.9.0 h1:DI8qLG5PEO0Mu1Oj51YFPqtx6I3qYXUAhJVJ/IzAVl0= -github.com/bufbuild/protocompile v0.9.0/go.mod h1:s89m1O8CqSYpyE/YaSGtg1r1YFMF5nLTwh4vlj6O444= +github.com/bufbuild/buf v1.32.0 h1:ycoJJzTFvpUV1+Hewcv0lja+OPzmVjORmvmL+Dxy9Sg= +github.com/bufbuild/buf v1.32.0/go.mod h1:trWT7vC+ujoZayhIVBOoU8JEOUVxWDFNs4+ixc5bLEA= +github.com/bufbuild/protocompile v0.13.1-0.20240510201809-752249dfc37f h1:cQLFPZXf32tbTLzUTg0n69Vi5kddhUiZMzpMzKRW0XU= +github.com/bufbuild/protocompile v0.13.1-0.20240510201809-752249dfc37f/go.mod h1:QJcgsTVPSBEMt+/3i2M/RpwjZc+DAXyPPDg0slmMk4c= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee h1:E6ET8YUcYJ1lAe6ctR3as7yqzW2BNItDFnaB5zQq/8M= +github.com/bufbuild/protoplugin v0.0.0-20240323223605-e2735f6c31ee/go.mod h1:HjGFxsck9RObrTJp2hXQZfWhPgZqnR6sR1U5fCA/Kus= github.com/bufbuild/protovalidate-go v0.6.2 h1:U/V3CGF0kPlR12v41rjO4DrYZtLcS4ZONLmWN+rJVCQ= github.com/bufbuild/protovalidate-go v0.6.2/go.mod h1:4BR3rKEJiUiTy+sqsusFn2ladOf0kYmA2Reo6BHSBgQ= github.com/bufbuild/protoyaml-go v0.1.9 h1:anV5UtF1Mlvkkgp4NWA6U/zOnJFng8Orq4Vf3ZUQHBU= @@ -383,12 +390,12 @@ github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v26.1.0+incompatible h1:+nwRy8Ocd8cYNQ60mozDDICICD8aoFGtlPXifX/UQ3Y= -github.com/docker/cli v26.1.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v26.1.2+incompatible h1:/MWZpUMMlr1hCGyquL8QNbL1hbivQ1kLuT3Z9s1Tlpg= +github.com/docker/cli v26.1.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v26.1.0+incompatible h1:W1G9MPNbskA6VZWL7b3ZljTh0pXI68FpINx0GKaOdaM= -github.com/docker/docker v26.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.2+incompatible h1:UVX5ZOrrfTGZZYEP+ZDq3Xn9PdHNXaSYMFPDumMqG2k= +github.com/docker/docker v26.1.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo= github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -519,8 +526,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid/v5 v5.1.0 h1:S5rqVKIigghZTCBKPCw0Y+bXkn26K3TB5mvQq2Ix8dk= -github.com/gofrs/uuid/v5 v5.1.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= +github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM= +github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= @@ -643,8 +650,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/pprof v0.0.0-20240422182052-72c8669ad3e7 h1:3q13T5NW3mlTJZM6B5UAsf2N5NYFbYWIyI3W8DlvBDU= -github.com/google/pprof v0.0.0-20240422182052-72c8669ad3e7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 h1:velgFPYr1X9TDwLIfkV7fWqsFlf7TeP11M/7kPd/dVI= +github.com/google/pprof v0.0.0-20240509144519-723abb6459b7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -765,8 +772,8 @@ github.com/jdx/go-netrc v1.0.0 h1:QbLMLyCZGj0NA8glAhxUpf1zDg6cxnWgMBbjq40W0gQ= github.com/jdx/go-netrc v1.0.0/go.mod h1:Gh9eFQJnoTNIRHXl2j5bJXA1u84hQWJWgGh569zF3v8= github.com/jgautheron/goconst v1.7.1 h1:VpdAG7Ca7yvvJk5n8dMwQhfEZJh95kl/Hl9S1OI5Jkk= github.com/jgautheron/goconst v1.7.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= -github.com/jhump/protoreflect v1.15.6 h1:WMYJbw2Wo+KOWwZFvgY0jMoVHM6i4XIvRs2RcBj5VmI= -github.com/jhump/protoreflect v1.15.6/go.mod h1:jCHoyYQIJnaabEYnbGwyo9hUqfyUMTbJw/tAut5t97E= +github.com/jhump/protoreflect v1.16.0 h1:54fZg+49widqXYQ0b+usAFHbMkBGR4PpXrsHc8+TBDg= +github.com/jhump/protoreflect v1.16.0/go.mod h1:oYPd7nPvcBw/5wlDfm/AVmU9zH9BgqGCI469pGxfj/8= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48= @@ -1005,8 +1012,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= -github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= +github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -1265,8 +1272,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= @@ -1773,10 +1780,10 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 h1:4HZJ3Xv1cmrJ+0aFo304Zn79ur1HMxptAE7aCPNLSqc= +google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1823,7 +1830,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=