Skip to content

Commit

Permalink
Merge branch 'main' into haris/schema-support
Browse files Browse the repository at this point in the history
  • Loading branch information
hariso authored May 16, 2024
2 parents 68479f6 + b45b05e commit 38d8870
Show file tree
Hide file tree
Showing 6 changed files with 156 additions and 11 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/conduitio/conduit
go 1.22.2

require (
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240510200701-cf1213ca7b09.1
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1
github.com/Masterminds/semver/v3 v3.2.1
github.com/Masterminds/sprig/v3 v3.2.3
github.com/NYTimes/gziphandler v1.1.1
Expand Down Expand Up @@ -52,7 +52,7 @@ require (
golang.org/x/tools v0.21.0
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be
google.golang.org/grpc v1.63.2
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.1
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
)
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
4d63.com/gochecknoglobals v0.2.1/go.mod h1:KRE8wtJB3CXCsb1xy421JfTHIIbmT3U5ruxw2Qu8fSU=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1 h1:2IGhRovxlsOIQgx2ekZWo4wTPAYpck41+18ICxs37is=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.33.0-20240401165935-b983156c5e99.1/go.mod h1:Tgn5bgL220vkFOI0KPStlcClPeOJzAv4uT+V8JXGUnw=
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240510200701-cf1213ca7b09.1 h1:S2x9psUon4BSyuOCKdcye+u0cPrfBstmTx9Nv1bR+KI=
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240510200701-cf1213ca7b09.1/go.mod h1:gUfBTdJmhwTJHoIaOQjy7xczHhqjkQfXSKs5TY4IsUY=
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1 h1:AhgzTziVBgbY6sCwiNmn4qwgQxmYrPt5jggS2JQOPtk=
buf.build/gen/go/grpc-ecosystem/grpc-gateway/protocolbuffers/go v1.34.1-20240515202120-acb39fc2531e.1/go.mod h1:gUfBTdJmhwTJHoIaOQjy7xczHhqjkQfXSKs5TY4IsUY=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
Expand Down Expand Up @@ -1806,8 +1806,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand Down
38 changes: 35 additions & 3 deletions pkg/plugin/processor/builtin/impl/webhook/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@ type httpConfig struct {
URL string `json:"request.url" validate:"required"`
// Method is the HTTP request method to be used.
Method string `json:"request.method" default:"GET"`
// The value of the `Content-Type` header.
// Deprecated: use `headers.Content-Type` instead.
ContentType string `json:"request.contentType" default:"application/json"`
// Headers to add to the request, use `headers.*` to specify the header and its value (e.g. `headers.Authorization: "Bearer key"`).
Headers map[string]string `json:"headers"`

// Maximum number of retries for an individual record when backing off following an error.
BackoffRetryCount float64 `json:"backoffRetry.count" default:"0" validate:"gt=-1"`
Expand All @@ -72,6 +74,29 @@ type httpConfig struct {
ResponseStatusRef string `json:"response.status"`
}

func (c *httpConfig) parseHeaders() error {
if c.Headers == nil {
c.Headers = make(map[string]string)
}

if c.ContentType == "" {
return nil // Nothing to replace in headers
}

for name, _ := range c.Headers {
if strings.ToLower(name) == "content-type" {
return cerrors.Errorf("Configuration error, cannot provide both \"request.contentType\" and \"headers.Content-Type\", use \"headers.Content-Type\" only.")
}
}

c.Headers["Content-Type"] = c.ContentType
// the ContentType field is deprecated,
// so we're preparing for completely removing it in a later release
c.ContentType = ""

return nil
}

type httpProcessor struct {
sdk.UnimplementedProcessor

Expand Down Expand Up @@ -108,6 +133,11 @@ func (p *httpProcessor) Configure(ctx context.Context, m map[string]string) erro
return cerrors.Errorf("failed parsing configuration: %w", err)
}

err = p.config.parseHeaders()
if err != nil {
return err
}

if p.config.ResponseBodyRef == p.config.ResponseStatusRef {
return cerrors.New("invalid configuration: response.body and response.status set to same field")
}
Expand Down Expand Up @@ -293,8 +323,10 @@ func (p *httpProcessor) buildRequest(ctx context.Context, r opencdc.Record) (*ht
return nil, cerrors.Errorf("error creating HTTP request: %w", err)
}

// todo make it possible to add more headers, e.g. auth headers etc.
req.Header.Set("Content-Type", p.config.ContentType)
// set header values
for key, val := range p.config.Headers {
req.Header.Set(key, val)
}

return req, nil
}
Expand Down
101 changes: 101 additions & 0 deletions pkg/plugin/processor/builtin/impl/webhook/http_config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// Copyright © 2024 Meroxa, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package webhook

import (
"github.com/matryer/is"
"testing"
)

func TestHTTPConfig_ValidateHeaders(t *testing.T) {
testCases := []struct {
name string
input httpConfig
wantConfig httpConfig
wantErr string
}{
{
name: "ContentType field present, header present",
input: httpConfig{
ContentType: "application/json",
Headers: map[string]string{
"Content-Type": "application/json",
},
},
wantErr: `Configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only.`,
},
{
name: "ContentType field present, header present, different case",
input: httpConfig{
ContentType: "application/json",
Headers: map[string]string{
"content-type": "application/json",
},
},
wantErr: `Configuration error, cannot provide both "request.contentType" and "headers.Content-Type", use "headers.Content-Type" only.`,
},
{
name: "ContentType field presents, header not present",
input: httpConfig{
ContentType: "application/json",
},
wantConfig: httpConfig{
Headers: map[string]string{
"Content-Type": "application/json",
},
},
},
{
name: "ContentType field not present, header present",
input: httpConfig{
Headers: map[string]string{
"Content-Type": "application/json",
},
},
wantConfig: httpConfig{
Headers: map[string]string{
"Content-Type": "application/json",
},
},
},
{
name: "ContentType field not present, header present, different case",
input: httpConfig{
Headers: map[string]string{
"content-type": "application/json",
},
},
wantConfig: httpConfig{
Headers: map[string]string{
"content-type": "application/json",
},
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
is := is.New(t)
err := tc.input.parseHeaders()

if tc.wantErr == "" {
is.Equal(tc.wantConfig, tc.input)
} else {
is.True(err != nil)
is.Equal(tc.wantErr, err.Error())
}
})
}
}
8 changes: 7 additions & 1 deletion pkg/plugin/processor/builtin/impl/webhook/http_paramgen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@
"type": "duration",
"validations": []
},
"headers.*": {
"default": "",
"description": "Headers to add to the request, use `headers.*` to specify the header and its value (e.g. `headers.Authorization: \"Bearer key\"`).",
"type": "string",
"validations": []
},
"request.body": {
"default": "",
"description": "Specifies which field from the input record should be used as the body in\nthe HTTP request.\n\nFor more information about the format, see [Referencing fields](https://conduit.io/docs/processors/referencing-fields).",
Expand All @@ -48,7 +54,7 @@
},
"request.contentType": {
"default": "application/json",
"description": "The value of the `Content-Type` header.",
"description": "Deprecated: use `headers.Content-Type` instead.",
"type": "string",
"validations": []
},
Expand Down

0 comments on commit 38d8870

Please sign in to comment.