From 28feaf4a5385473c82bc565f96bb2c0eb213323a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Fri, 24 May 2024 17:23:36 +0200 Subject: [PATCH] Use cpluginv2 and conduit commons (#130) * use cpluginv2 and conduit commons * organize imports * update to new cplugin types --- acceptance_testing.go | 100 +-- benchmark.go | 6 +- cmd/paramgen/README.md | 170 ----- cmd/paramgen/internal/paramgen.go | 708 ------------------ cmd/paramgen/internal/paramgen_test.go | 207 ----- cmd/paramgen/internal/template.go | 138 ---- cmd/paramgen/internal/template_test.go | 145 ---- cmd/paramgen/internal/testdata/basic/go.mod | 3 - cmd/paramgen/internal/testdata/basic/specs.go | 71 -- .../internal/testdata/complex/global.go | 25 - cmd/paramgen/internal/testdata/complex/go.mod | 3 - .../testdata/complex/internal/global.go | 31 - .../internal/testdata/complex/specs.go | 40 - .../internal/testdata/complex/specs_test.go | 23 - .../internal/testdata/complex/tools.go | 19 - .../internal/testdata/invalid1/go.mod | 3 - .../internal/testdata/invalid1/specs.go | 22 - .../internal/testdata/invalid2/go.mod | 3 - .../internal/testdata/invalid2/specs.go | 19 - cmd/paramgen/internal/testdata/tags/go.mod | 3 - cmd/paramgen/internal/testdata/tags/specs.go | 29 - cmd/paramgen/main.go | 90 --- destination.go | 156 ++-- destination_middleware.go | 80 +- destination_middleware_test.go | 32 +- destination_test.go | 244 +++--- error.go | 4 - go.mod | 25 +- go.sum | 53 +- kafkaconnect/schema_test.go | 3 +- metadata.go | 241 ------ mock_destination_test.go | 8 +- mock_source_test.go | 14 +- operation_string.go | 27 - record.go | 177 ----- record_formatter.go => record_serializer.go | 73 +- ...atter_test.go => record_serializer_test.go | 121 +-- record_test.go | 44 -- serve.go | 10 +- source.go | 171 ++--- source_test.go | 211 ++---- specifier.go | 92 +-- specifier_test.go | 6 +- stream_test.go | 152 ++++ unimplemented.go | 17 +- util.go | 4 +- validation.go | 137 ---- validation_test.go | 538 ------------- 48 files changed, 743 insertions(+), 3755 deletions(-) delete mode 100644 cmd/paramgen/README.md delete mode 100644 cmd/paramgen/internal/paramgen.go delete mode 100644 cmd/paramgen/internal/paramgen_test.go delete mode 100644 cmd/paramgen/internal/template.go delete mode 100644 cmd/paramgen/internal/template_test.go delete mode 100644 cmd/paramgen/internal/testdata/basic/go.mod delete mode 100644 cmd/paramgen/internal/testdata/basic/specs.go delete mode 100644 cmd/paramgen/internal/testdata/complex/global.go delete mode 100644 cmd/paramgen/internal/testdata/complex/go.mod delete mode 100644 cmd/paramgen/internal/testdata/complex/internal/global.go delete mode 100644 cmd/paramgen/internal/testdata/complex/specs.go delete mode 100644 cmd/paramgen/internal/testdata/complex/specs_test.go delete mode 100644 cmd/paramgen/internal/testdata/complex/tools.go delete mode 100644 cmd/paramgen/internal/testdata/invalid1/go.mod delete mode 100644 cmd/paramgen/internal/testdata/invalid1/specs.go delete mode 100644 cmd/paramgen/internal/testdata/invalid2/go.mod delete mode 100644 cmd/paramgen/internal/testdata/invalid2/specs.go delete mode 100644 cmd/paramgen/internal/testdata/tags/go.mod delete mode 100644 cmd/paramgen/internal/testdata/tags/specs.go delete mode 100644 cmd/paramgen/main.go delete mode 100644 metadata.go delete mode 100644 operation_string.go delete mode 100644 record.go rename record_formatter.go => record_serializer.go (74%) rename record_formatter_test.go => record_serializer_test.go (70%) delete mode 100644 record_test.go create mode 100644 stream_test.go delete mode 100644 validation.go delete mode 100644 validation_test.go diff --git a/acceptance_testing.go b/acceptance_testing.go index ae9e8b5d..830d5860 100644 --- a/acceptance_testing.go +++ b/acceptance_testing.go @@ -29,6 +29,8 @@ import ( "testing" "time" + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" @@ -88,7 +90,7 @@ type AcceptanceTestDriver interface { // The generated record will contain mixed data types in the field Key and // Payload (i.e. RawData and StructuredData), unless configured otherwise // (see ConfigurableAcceptanceTestDriverConfig.GenerateDataType). - GenerateRecord(*testing.T, Operation) Record + GenerateRecord(*testing.T, opencdc.Operation) opencdc.Record // WriteToSource receives a slice of records that should be prepared in the // 3rd party system so that the source will read them. The returned slice @@ -97,7 +99,7 @@ type AcceptanceTestDriver interface { // It is encouraged for the driver to return the same slice, unless there is // no way to write the records to the 3rd party system, then the returning // slice should contain the expected records a source should read. - WriteToSource(*testing.T, []Record) []Record + WriteToSource(*testing.T, []opencdc.Record) []opencdc.Record // ReadFromDestination should return a slice with the records that were // written to the destination. The slice will be used to verify the // destination has successfully executed writes. @@ -105,7 +107,7 @@ type AcceptanceTestDriver interface { // destination. These will be compared to the returned slice of records. It // is encouraged for the driver to only touch the input records to change // the order of records and to not change the records themselves. - ReadFromDestination(*testing.T, []Record) []Record + ReadFromDestination(*testing.T, []opencdc.Record) []opencdc.Record // ReadTimeout controls the time the test should wait for a read operation // to return before it considers the operation as failed. @@ -230,7 +232,7 @@ func (d ConfigurableAcceptanceTestDriver) GoleakOptions(_ *testing.T) []goleak.O return d.Config.GoleakOptions } -func (d ConfigurableAcceptanceTestDriver) GenerateRecord(t *testing.T, op Operation) Record { +func (d ConfigurableAcceptanceTestDriver) GenerateRecord(t *testing.T, op opencdc.Operation) opencdc.Record { // TODO we currently only generate records with operation "create" and // "snapshot", because this is the only operation we know all connectors // should be able to handle. We should make acceptance tests more @@ -238,12 +240,12 @@ func (d ConfigurableAcceptanceTestDriver) GenerateRecord(t *testing.T, op Operat // specifically "update" and "delete". // Once we generate different operations we need to adjust how we compare // records! - return Record{ - Position: Position(d.randString(32)), // position doesn't matter, as long as it's unique + return opencdc.Record{ + Position: opencdc.Position(d.randString(32)), // position doesn't matter, as long as it's unique Operation: op, Metadata: map[string]string{d.randString(32): d.randString(32)}, Key: d.GenerateData(t), - Payload: Change{ + Payload: opencdc.Change{ Before: nil, After: d.GenerateData(t), }, @@ -253,7 +255,7 @@ func (d ConfigurableAcceptanceTestDriver) GenerateRecord(t *testing.T, op Operat // GenerateData generates either RawData or StructuredData depending on the // configured data type (see // ConfigurableAcceptanceTestDriverConfig.GenerateDataType). -func (d ConfigurableAcceptanceTestDriver) GenerateData(t *testing.T) Data { +func (d ConfigurableAcceptanceTestDriver) GenerateData(t *testing.T) opencdc.Data { rand := d.getRand() gen := d.Config.GenerateDataType @@ -263,9 +265,9 @@ func (d ConfigurableAcceptanceTestDriver) GenerateData(t *testing.T) Data { switch gen { case GenerateRawData: - return RawData(d.randString(rand.Intn(1024) + 32)) + return opencdc.RawData(d.randString(rand.Intn(1024) + 32)) case GenerateStructuredData: - data := StructuredData{} + data := opencdc.StructuredData{} for { data[d.randString(rand.Intn(1024)+32)] = d.GenerateValue(t) if rand.Int63()%2 == 0 { @@ -355,7 +357,7 @@ func (d ConfigurableAcceptanceTestDriver) randString(n int) string { // destination. It is expected that the destination is writing to the same // location the source is reading from. If the connector does not implement a // destination the function will fail the test. -func (d ConfigurableAcceptanceTestDriver) WriteToSource(t *testing.T, records []Record) []Record { +func (d ConfigurableAcceptanceTestDriver) WriteToSource(t *testing.T, records []opencdc.Record) []opencdc.Record { if d.Connector().NewDestination == nil { t.Fatal("connector is missing the field NewDestination, either implement the destination or overwrite the driver method Write") } @@ -394,7 +396,7 @@ func (d ConfigurableAcceptanceTestDriver) WriteToSource(t *testing.T, records [] // the source. It is expected that the destination is writing to the same // location the source is reading from. If the connector does not implement a // source the function will fail the test. -func (d ConfigurableAcceptanceTestDriver) ReadFromDestination(t *testing.T, records []Record) []Record { +func (d ConfigurableAcceptanceTestDriver) ReadFromDestination(t *testing.T, records []opencdc.Record) []opencdc.Record { if d.Connector().NewSource == nil { t.Fatal("connector is missing the field NewSource, either implement the source or overwrite the driver method Read") } @@ -429,7 +431,7 @@ func (d ConfigurableAcceptanceTestDriver) ReadFromDestination(t *testing.T, reco Max: time.Second, // 8 tries } - output := make([]Record, 0, len(records)) + output := make([]opencdc.Record, 0, len(records)) for i := 0; i < cap(output); i++ { // now try to read from the source readCtx, readCancel := context.WithTimeout(ctx, d.ReadTimeout()) @@ -458,7 +460,7 @@ func (d ConfigurableAcceptanceTestDriver) ReadFromDestination(t *testing.T, reco readCtx, readCancel := context.WithTimeout(ctx, d.ReadTimeout()) defer readCancel() r, err := src.Read(readCtx) - is.Equal(Record{}, r) // record should be empty + is.Equal(opencdc.Record{}, r) // record should be empty is.True(errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrBackoffRetry)) return output @@ -585,9 +587,9 @@ func (a acceptanceTest) TestSource_Configure_RequiredParams(t *testing.T) { origCfg := a.driver.SourceConfig(t) for name, p := range srcSpec.Parameters() { - isRequired := p.Required + isRequired := false for _, v := range p.Validations { - if _, ok := v.(ValidationRequired); ok { + if _, ok := v.(config.ValidationRequired); ok { isRequired = true break } @@ -620,7 +622,7 @@ func (a acceptanceTest) TestSource_Open_ResumeAtPositionSnapshot(t *testing.T) { // Write expectations before source is started, this means the source will // have to first read the existing data (i.e. snapshot), but we will // interrupt it and try to resume. - want := a.driver.WriteToSource(t, a.generateRecords(t, OperationSnapshot, 10)) + want := a.driver.WriteToSource(t, a.generateRecords(t, opencdc.OperationSnapshot, 10)) source, sourceCleanup := a.openSource(ctx, t, nil) // listen from beginning defer sourceCleanup() @@ -681,7 +683,7 @@ func (a acceptanceTest) TestSource_Open_ResumeAtPositionCDC(t *testing.T) { // Write expectations after source is open, this means the source is already // listening to ongoing changes (i.e. CDC), we will interrupt it and try to // resume. - want := a.driver.WriteToSource(t, a.generateRecords(t, OperationCreate, 10)) + want := a.driver.WriteToSource(t, a.generateRecords(t, opencdc.OperationCreate, 10)) // read all records, but ack only half of them got, err := a.readMany(ctx, t, source, len(want)) @@ -718,7 +720,7 @@ func (a acceptanceTest) TestSource_Read_Success(t *testing.T) { defer goleak.VerifyNone(t, a.driver.GoleakOptions(t)...) positions := make(map[string]bool) - isUniquePositions := func(t *testing.T, records []Record) { + isUniquePositions := func(t *testing.T, records []opencdc.Record) { is := is.New(t) for _, r := range records { is.True(!positions[string(r.Position)]) @@ -727,7 +729,7 @@ func (a acceptanceTest) TestSource_Read_Success(t *testing.T) { } // write expectation before source exists - want := a.driver.WriteToSource(t, a.generateRecords(t, OperationSnapshot, 10)) + want := a.driver.WriteToSource(t, a.generateRecords(t, opencdc.OperationSnapshot, 10)) source, sourceCleanup := a.openSource(ctx, t, nil) // listen from beginning defer sourceCleanup() @@ -745,7 +747,7 @@ func (a acceptanceTest) TestSource_Read_Success(t *testing.T) { // while connector is running write more data and make sure the connector // detects it - want = a.driver.WriteToSource(t, a.generateRecords(t, OperationCreate, 20)) + want = a.driver.WriteToSource(t, a.generateRecords(t, opencdc.OperationCreate, 20)) t.Run("cdc", func(t *testing.T) { is := is.New(t) @@ -771,7 +773,7 @@ func (a acceptanceTest) TestSource_Read_Timeout(t *testing.T) { readCtx, cancel := context.WithTimeout(ctx, a.driver.ReadTimeout()) defer cancel() r, err := a.readWithBackoffRetry(readCtx, t, source) - is.Equal(Record{}, r) // record should be empty + is.Equal(opencdc.Record{}, r) // record should be empty is.True(errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrBackoffRetry)) } @@ -819,9 +821,9 @@ func (a acceptanceTest) TestDestination_Configure_RequiredParams(t *testing.T) { origCfg := a.driver.DestinationConfig(t) for name, p := range destSpec.Parameters() { - isRequired := p.Required + isRequired := false for _, v := range p.Validations { - if _, ok := v.(ValidationRequired); ok { + if _, ok := v.(config.ValidationRequired); ok { isRequired = true break } @@ -854,7 +856,7 @@ func (a acceptanceTest) TestDestination_Write_Success(t *testing.T) { dest, cleanup := a.openDestination(ctx, t) defer cleanup() - want := a.generateRecords(t, OperationSnapshot, 20) + want := a.generateRecords(t, opencdc.OperationSnapshot, 20) writeCtx, cancel := context.WithTimeout(ctx, a.driver.WriteTimeout()) defer cancel() @@ -893,7 +895,7 @@ func (a acceptanceTest) cloneConfig(orig map[string]string) map[string]string { return cloned } -func (a acceptanceTest) openSource(ctx context.Context, t *testing.T, pos Position) (source Source, cleanup func()) { +func (a acceptanceTest) openSource(ctx context.Context, t *testing.T, pos opencdc.Position) (source Source, cleanup func()) { is := is.New(t) source = a.driver.Connector().NewSource() @@ -941,16 +943,16 @@ func (a acceptanceTest) openDestination(ctx context.Context, t *testing.T) (dest return dest, cleanup } -func (a acceptanceTest) generateRecords(t *testing.T, op Operation, count int) []Record { - records := make([]Record, count) +func (a acceptanceTest) generateRecords(t *testing.T, op opencdc.Operation, count int) []opencdc.Record { + records := make([]opencdc.Record, count) for i := range records { records[i] = a.driver.GenerateRecord(t, op) } return records } -func (a acceptanceTest) readMany(ctx context.Context, t *testing.T, source Source, limit int) ([]Record, error) { - var got []Record +func (a acceptanceTest) readMany(ctx context.Context, t *testing.T, source Source, limit int) ([]opencdc.Record, error) { + var got []opencdc.Record for i := 0; i < limit; i++ { readCtx, cancel := context.WithTimeout(ctx, a.driver.ReadTimeout()) defer cancel() @@ -965,7 +967,7 @@ func (a acceptanceTest) readMany(ctx context.Context, t *testing.T, source Sourc return got, nil } -func (a acceptanceTest) readWithBackoffRetry(ctx context.Context, t *testing.T, source Source) (Record, error) { +func (a acceptanceTest) readWithBackoffRetry(ctx context.Context, t *testing.T, source Source) (opencdc.Record, error) { b := &backoff.Backoff{ Factor: 2, Min: time.Millisecond * 100, @@ -979,7 +981,7 @@ func (a acceptanceTest) readWithBackoffRetry(ctx context.Context, t *testing.T, select { case <-ctx.Done(): // return error - return Record{}, ctx.Err() + return opencdc.Record{}, ctx.Err() case <-time.After(b.Duration()): continue } @@ -989,7 +991,7 @@ func (a acceptanceTest) readWithBackoffRetry(ctx context.Context, t *testing.T, } // isEqualRecords compares two record slices and disregards their order. -func (a acceptanceTest) isEqualRecords(is *is.I, want, got []Record) { +func (a acceptanceTest) isEqualRecords(is *is.I, want, got []opencdc.Record) { is.Equal(len(want), len(got)) // record number did not match if len(want) == 0 { @@ -1004,14 +1006,14 @@ func (a acceptanceTest) isEqualRecords(is *is.I, want, got []Record) { // retrieves them as a whole in the payload // this is valid behavior, we need to adjust the expectations for i, wantRec := range want { - want[i] = Record{ + want[i] = opencdc.Record{ Position: nil, - Operation: OperationSnapshot, // we allow operations Snapshot or Create - Metadata: nil, // no expectation for metadata - Key: got[i].Key, // no expectation for key - Payload: Change{ + Operation: opencdc.OperationSnapshot, // we allow operations Snapshot or Create + Metadata: nil, // no expectation for metadata + Key: got[i].Key, // no expectation for key + Payload: opencdc.Change{ Before: nil, - After: RawData(wantRec.Bytes()), // the payload should contain the whole expected record + After: opencdc.RawData(wantRec.Bytes()), // the payload should contain the whole expected record }, } } @@ -1031,7 +1033,7 @@ func (a acceptanceTest) isEqualRecords(is *is.I, want, got []Record) { // - It then sorts only the want slice using the output of Record.Bytes(). // This assumes that the destination writes whole records and not only the // payload. It does not check if the first records match after this. -func (a acceptanceTest) sortMatchingRecords(want, got []Record) { +func (a acceptanceTest) sortMatchingRecords(want, got []opencdc.Record) { sort.Slice(want, func(i, j int) bool { return string(want[i].Payload.After.Bytes()) < string(want[j].Payload.After.Bytes()) }) @@ -1051,9 +1053,9 @@ func (a acceptanceTest) sortMatchingRecords(want, got []Record) { }) } -func (a acceptanceTest) isEqualRecord(is *is.I, want, got Record) { - if want.Operation == OperationSnapshot && - got.Operation == OperationCreate { +func (a acceptanceTest) isEqualRecord(is *is.I, want, got opencdc.Record) { + if want.Operation == opencdc.OperationSnapshot && + got.Operation == opencdc.OperationCreate { // This is a special case and we accept it. Not all connectors will // create records with operation "snapshot", but they will still able to // produce records that were written before the source was open in @@ -1078,18 +1080,18 @@ func (a acceptanceTest) isEqualRecord(is *is.I, want, got Record) { } } -func (a acceptanceTest) isEqualChange(is *is.I, want, got Change) { +func (a acceptanceTest) isEqualChange(is *is.I, want, got opencdc.Change) { a.isEqualData(is, want.Before, got.Before) a.isEqualData(is, want.After, got.After) } // isEqualData will match the two data objects in their entirety if the types // match or only the byte slice content if types differ. -func (a acceptanceTest) isEqualData(is *is.I, want, got Data) { - _, wantIsRaw := want.(RawData) - _, gotIsRaw := got.(RawData) - _, wantIsStructured := want.(StructuredData) - _, gotIsStructured := got.(StructuredData) +func (a acceptanceTest) isEqualData(is *is.I, want, got opencdc.Data) { + _, wantIsRaw := want.(opencdc.RawData) + _, gotIsRaw := got.(opencdc.RawData) + _, wantIsStructured := want.(opencdc.StructuredData) + _, gotIsStructured := got.(opencdc.StructuredData) if (wantIsRaw && gotIsRaw) || (wantIsStructured && gotIsStructured) || diff --git a/benchmark.go b/benchmark.go index b679f7c8..b5f33379 100644 --- a/benchmark.go +++ b/benchmark.go @@ -19,6 +19,8 @@ import ( "sync" "testing" "time" + + "github.com/conduitio/conduit-commons/opencdc" ) // BenchmarkSource is a benchmark that any source implementation can run to figure @@ -84,7 +86,7 @@ func (bm *benchmarkSource) Run(b *testing.B) { } }) - acks := make(chan Record, b.N) // huge buffer so we don't delay reads + acks := make(chan opencdc.Record, b.N) // huge buffer so we don't delay reads var wg sync.WaitGroup wg.Add(1) go bm.acker(b, acks, &wg) @@ -127,7 +129,7 @@ func (bm *benchmarkSource) Run(b *testing.B) { bm.reportMetrics(b) } -func (bm *benchmarkSource) acker(b *testing.B, c <-chan Record, wg *sync.WaitGroup) { +func (bm *benchmarkSource) acker(b *testing.B, c <-chan opencdc.Record, wg *sync.WaitGroup) { defer wg.Done() ctx := context.Background() diff --git a/cmd/paramgen/README.md b/cmd/paramgen/README.md deleted file mode 100644 index adf70d6b..00000000 --- a/cmd/paramgen/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# ParamGen - -ParamGen is a conduit tool that generates the code to return the parameters map from a certain Go struct. - -## Installation - -Once you have installed Go, install the paramgen tool. - -**Note:** If you have not done so already be sure to add `$GOPATH/bin` to your `PATH`. - -``` -go install github.com/conduitio/conduit-connector-sdk/cmd/paramgen@latest -``` - -## Usage - -ParamGen has one required argument, which is the struct name, and two optional flags for the path and the output file name. - -``` -paramgen [-path] [-output] structName -``` - -Example: - -``` -paramgen -path=./source -output=source_params.go SourceConfig -``` - -This example will search for a struct called `SourceConfig` in the path `./source`, it will create a parameter map of -only the exported fields, and generate the code to return this map in the file `source_params.go` under the same folder. - -### Parameter Tags - -In order to give your parameter a name, a default value, or add some validations to it, tags are the way to go. -We have three tags that can be parsed: - -1. `json`: this tag is used to rename the parameter. - - ```go - Name string `json:"first-name"` - ``` - -2. `default`: sets the default value for the parameter. - - ```go - Name string `default:"conduit"` - ``` - -3. `validate`: adds builtin validations to the parameter, these validations will be executed by conduit once a connector - is configured. Validations are separated by a comma, and have 6 main types: - * `required`: a boolean tag to indicate if a field is required or not. If it is added to the validate tag without a - value, then we assume the field is required. - - ```go - NameRequired string `validate:"required"` - NameRequired2 string `validate:"required=true"` - NameNotRequired string `validate:"required=false"` - ``` - - * `lt` or `less-than`: takes an int or a float value, indicated that the parameter should be less than the value provided. - * `gt` or `greater-than`: takes an int or a float value, indicated that the parameter should be greater than the value provided. - - ```go - Age int `validate:"gt=0,lt=200"` - Age2 float `validate:"greater-than=0,less-than=200.2"` - ``` - - * `inclusion`: validates that the parameter value is included in a specified list, this list values are separated - using a pipe character `|`. - - ```go - Gender string `validate:"inclusion=male|female|other"` - ``` - - * `exclusion`: validates that the parameter value is NOT included in a specified list, this list values are separated - using a pipe character `|`. - - ```go - Color string `validate:"exclusion=red|green"` - ``` - - * `regex`: validates that the parameter value matches the regex pattern. - - ```go - Email string `validate:"regex=^[\w-\.]+@([\w-]+\.)+[\w-]{2,4}$"` - ``` - -## Example - -Assume we have this configuration struct: - -```go -package source -type SourceConfig struct { - InnerConfig - // Param1 my param1 description - Param1 int `validate:"required,gt=0,lt=100" default:"10"` - // comment about param2 - Param2 bool `validate:"inclusion=true|t,exclusion=false|f" default:"t"` - Param3 string `validate:"regex=.*" default:"yes"` - - // this will be ignored because it's not exported - privateParam string -} - -type InnerConfig struct { - Name string `validate:"required" json:"my-name"` -} -``` - -And you call ParamGen: - -``` -paramgen -path ./source SourceConfig -``` - -A file called `paramgen.go` will be created under `./source`: - -```go -// Code generated by ParamGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit-connector-sdk/cmd/paramgen - -package source - -import ( - "regexp" - - sdk "github.com/conduitio/conduit-connector-sdk" -) - -func (SourceConfig) Parameters() map[string]sdk.Parameter { - return map[string]sdk.Parameter{ - "innerConfig.my-name": { - Default: "", - Description: "", - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - }, - }, - "param1": { - Default: "10", - Description: "param1 my param1 description", - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - sdk.ValidationGreaterThan{Value: 0}, - sdk.ValidationLessThan{Value: 100}, - }, - }, - "param2": { - Default: "t", - Description: "comment about param2", - Type: sdk.ParameterTypeBool, - Validations: []sdk.Validation{ - sdk.ValidationInclusion{List: []string{"true", "t"}}, - sdk.ValidationExclusion{List: []string{"false", "f"}}, - }, - }, - "param3": { - Default: "yes", - Description: "", - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{ - sdk.ValidationRegex{Regex: regexp.MustCompile(".*")}, - }, - }, - } -} -``` diff --git a/cmd/paramgen/internal/paramgen.go b/cmd/paramgen/internal/paramgen.go deleted file mode 100644 index f0820444..00000000 --- a/cmd/paramgen/internal/paramgen.go +++ /dev/null @@ -1,708 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "encoding/json" - "fmt" - "go/ast" - "go/parser" - "go/token" - "io/fs" - "os/exec" - "reflect" - "regexp" - "strconv" - "strings" - "unicode" - - sdk "github.com/conduitio/conduit-connector-sdk" - "golang.org/x/exp/maps" -) - -const ( - tagParamName = "json" - tagParamDefault = "default" - tagParamValidate = "validate" - - validationRequired = "required" - validationLT = "lt" - validationLessThan = "less-than" - validationGT = "gt" - validationGreaterThan = "greater-than" - validationInclusion = "inclusion" - validationExclusion = "exclusion" - validationRegex = "regex" - - tagSeparator = "," - validateSeparator = "=" - listSeparator = "|" - fieldSeparator = "." -) - -// ParseParameters parses the struct into a map of parameter, requires the folder path that has the struct, and the -// struct name -func ParseParameters(path string, name string) (map[string]sdk.Parameter, string, error) { - mod, err := parseModule(path) - if err != nil { - return nil, "", fmt.Errorf("error parsing module: %w", err) - } - pkg, err := parsePackage(path) - if err != nil { - return nil, "", fmt.Errorf("error parsing package: %w", err) - } - myStruct, file, err := findStruct(pkg, name) - if err != nil { - return nil, "", err - } - - return (¶meterParser{ - pkg: pkg, - mod: mod, - file: file, - imports: map[string]*ast.Package{}, - }).Parse(myStruct) -} - -type module struct { - Path string // module path - Dir string // directory holding files for this module, if any - Error *moduleError // error loading module -} - -type moduleError struct { - Err string // the error itself -} - -func parseModule(path string) (module, error) { - cmd := exec.Command("go", "list", "-m", "-json") - cmd.Dir = path - stdout, err := cmd.StdoutPipe() - - if err != nil { - return module{}, fmt.Errorf("error piping stdout of go list command: %w", err) - } - if err := cmd.Start(); err != nil { - return module{}, fmt.Errorf("error starting go list command: %w", err) - } - var mod module - if err := json.NewDecoder(stdout).Decode(&mod); err != nil { - return module{}, fmt.Errorf("error decoding go list output: %w", err) - } - if err := cmd.Wait(); err != nil { - return module{}, fmt.Errorf("error running command %q: %w", cmd.String(), err) - } - if mod.Error != nil { - return module{}, fmt.Errorf("error loading module: %s", mod.Error.Err) - } - return mod, nil -} - -func parsePackage(path string) (*ast.Package, error) { - fset := token.NewFileSet() - filterTests := func(info fs.FileInfo) bool { - return !strings.HasSuffix(info.Name(), "_test.go") - } - pkgs, err := parser.ParseDir(fset, path, filterTests, parser.ParseComments) - if err != nil { - return nil, fmt.Errorf("couldn't parse directory %s: %w", path, err) - } - // Make sure they are all in one package. - if len(pkgs) == 0 { - return nil, fmt.Errorf("no source-code package in directory %s", path) - } - // Ignore files with go:build constraint set to "tools" (common pattern in - // Conduit connectors). - for pkgName, pkg := range pkgs { - maps.DeleteFunc(pkg.Files, func(_ string, f *ast.File) bool { - return hasBuildConstraint(f, "tools") - }) - if len(pkg.Files) == 0 { - delete(pkgs, pkgName) - } - } - if len(pkgs) > 1 { - return nil, fmt.Errorf("multiple packages %v in directory %s", maps.Keys(pkgs), path) - } - for _, v := range pkgs { - return v, nil // return first package - } - panic("unreachable") -} - -// hasBuildConstraint is a very naive way to check if a file has a build -// constraint. It is sufficient for our use case. -func hasBuildConstraint(f *ast.File, constraint string) bool { - text := fmt.Sprintf("//go:build %s", constraint) - for _, cg := range f.Comments { - for _, c := range cg.List { - if c.Text == text { - return true - } - } - } - return false -} - -func findStruct(pkg *ast.Package, name string) (*ast.StructType, *ast.File, error) { - var structType *ast.StructType - var file *ast.File - for _, f := range pkg.Files { - ast.Inspect(f, func(n ast.Node) bool { - // Check if the node is a struct declaration - if typeSpec, ok := n.(*ast.TypeSpec); ok && typeSpec.Name.String() == name { - structType, ok = typeSpec.Type.(*ast.StructType) - if !ok { - // Node is not a struct declaration - return true - } - file = f - // stop iterating - return false - } - // Return true to continue iterating over the ast.File - return true - }) - } - if file == nil { - return nil, nil, fmt.Errorf("struct %q was not found in the package %q", name, pkg.Name) - } - return structType, file, nil -} - -type parameterParser struct { - // pkg holds the current package we are working with - pkg *ast.Package - // file holds the current file we are working with - file *ast.File - - mod module - - imports map[string]*ast.Package -} - -func (p *parameterParser) Parse(structType *ast.StructType) (map[string]sdk.Parameter, string, error) { - pkgName := p.pkg.Name - - parameters, err := p.parseStructType(structType, nil) - if err != nil { - return nil, "", err - } - - return parameters, pkgName, nil -} - -func (p *parameterParser) parseIdent(ident *ast.Ident, field *ast.Field) (params map[string]sdk.Parameter, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("[parseIdent] %w", err) - } - }() - - if p.isBuiltinType(ident.Name) { - // builtin type, that's a parameter - t := p.getParamType(ident) - name, param, err := p.parseSingleParameter(field, t) - if err != nil { - return nil, err - } - return map[string]sdk.Parameter{name: param}, nil - } - - if ident.Obj == nil { - // need to find the identifier in another file - ts, file, err := p.findType(p.pkg, ident.Name) - if err != nil { - return nil, err - } - - // change the type for simplicity - ident.Obj = &ast.Object{ - Name: ident.Name, - Decl: ts, - } - - // back up current file and replace it because we are now working with - // another file, we want to revert this once we are done parsing this type - backupFile := p.file - p.file = file - defer func() { - p.file = backupFile - }() - } - - switch v := ident.Obj.Decl.(type) { - case *ast.TypeSpec: - return p.parseTypeSpec(v, field) - default: - return nil, fmt.Errorf("unexpected type: %T", ident.Obj.Decl) - } -} - -func (p *parameterParser) parseTypeSpec(ts *ast.TypeSpec, f *ast.Field) (params map[string]sdk.Parameter, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("[parseTypeSpec] %w", err) - } - }() - - switch v := ts.Type.(type) { - case *ast.StructType: - return p.parseStructType(v, f) - case *ast.SelectorExpr: - return p.parseSelectorExpr(v, f) - case *ast.Ident: - return p.parseIdent(v, f) - case *ast.MapType: - return p.parseMapType(v, f) - default: - return nil, fmt.Errorf("unexpected type: %T", ts.Type) - } -} - -func (p *parameterParser) parseStructType(st *ast.StructType, f *ast.Field) (params map[string]sdk.Parameter, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("[parseStructType] %w", err) - } - }() - - for _, f := range st.Fields.List { - fieldParams, err := p.parseField(f) - if err != nil { - return nil, fmt.Errorf("error parsing field %q: %w", f.Names[0].Name, err) - } - if params == nil { - params = fieldParams - continue - } - for k, v := range fieldParams { - if _, ok := params[k]; ok { - return nil, fmt.Errorf("parameter %q is defined twice", k) - } - params[k] = v - } - } - if f != nil { - // attach prefix of field in which this struct type is declared - params = p.attachPrefix(f, params) - } - return params, nil -} - -// parse tags, defaults and stuff -func (p *parameterParser) parseField(f *ast.Field) (params map[string]sdk.Parameter, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("[parseField] %w", err) - } - }() - - if len(f.Names) == 1 && !f.Names[0].IsExported() { - return nil, nil // ignore unexported fields - } - - switch v := f.Type.(type) { - case *ast.Ident: - // identifier (builtin type or type in same package) - return p.parseIdent(v, f) - case *ast.StructType: - // nested type - return p.parseStructType(v, f) - case *ast.SelectorExpr: - return p.parseSelectorExpr(v, f) - case *ast.MapType: - return p.parseMapType(v, f) - case *ast.ArrayType: - strType := fmt.Sprintf("%s", v.Elt) - if !p.isBuiltinType(strType) && !strings.Contains(strType, "time Duration") { - return nil, fmt.Errorf("unsupported slice type: %s", strType) - } - - name, param, err := p.parseSingleParameter(f, sdk.ParameterTypeString) - if err != nil { - return nil, err - } - return map[string]sdk.Parameter{name: param}, nil - default: - return nil, fmt.Errorf("unknown type: %T", f.Type) - } -} - -func (p *parameterParser) parseMapType(mt *ast.MapType, f *ast.Field) (params map[string]sdk.Parameter, err error) { - if fmt.Sprintf("%s", mt.Key) != "string" { - return nil, fmt.Errorf("unsupported map key type: %s", mt.Key) - } - - // parse map value as if it was a field - var tmpParams map[string]sdk.Parameter - switch val := mt.Value.(type) { - case *ast.Ident: - // identifier (builtin type or type in same package) - tmpParams, err = p.parseIdent(val, f) - case *ast.StructType: - // nested type - tmpParams, err = p.parseStructType(val, f) - case *ast.SelectorExpr: - tmpParams, err = p.parseSelectorExpr(val, f) - } - if err != nil { - return nil, err - } - - // inject wildcard - params = make(map[string]sdk.Parameter, len(tmpParams)) - for k, p := range tmpParams { - index := strings.Index(k, ".") - if index == -1 { - index = len(k) - } - name := k[:index] + ".*" - if index < len(k) { - name += k[index:] - } - params[name] = p - } - return params, nil -} - -func (p *parameterParser) parseSelectorExpr(se *ast.SelectorExpr, f *ast.Field) (params map[string]sdk.Parameter, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("[parseSelectorExpr] %w", err) - } - }() - - imp, err := p.findImportSpec(se) - if err != nil { - return nil, err - } - - if impPath := strings.Trim(imp.Path.Value, `"`); impPath == "time" && se.Sel.Name == "Duration" { - // we allow the duration type - name, param, err := p.parseSingleParameter(f, sdk.ParameterTypeDuration) - if err != nil { - return nil, err - } - return map[string]sdk.Parameter{name: param}, nil - } - - // first find package - pkg, err := p.findPackage(imp.Path.Value) - if err != nil { - return nil, err - } - - // now find requested type in that package - ts, file, err := p.findType(pkg, se.Sel.Name) - if err != nil { - return nil, err - } - - // back up current file and replace it because we are now working with - // another file, we want to revert this once we are done parsing this type - backupFile := p.file - backupPkg := p.pkg - p.file = file - p.pkg = pkg - defer func() { - p.file = backupFile - p.pkg = backupPkg - }() - - return p.parseTypeSpec(ts, f) -} - -func (p *parameterParser) findPackage(importPath string) (*ast.Package, error) { - // first cleanup string - importPath = strings.Trim(importPath, `"`) - - if !strings.HasPrefix(importPath, p.mod.Path) { - // we only allow types declared in the same module - return nil, fmt.Errorf("we do not support parameters from package %v (please use builtin types or time.Duration)", importPath) - } - - if pkg, ok := p.imports[importPath]; ok { - // it's cached already - return pkg, nil - } - - pkgDir := p.mod.Dir + strings.TrimPrefix(importPath, p.mod.Path) - pkg, err := parsePackage(pkgDir) - if err != nil { - return nil, fmt.Errorf("could not parse package dir %q: %w", pkgDir, err) - } - - // cache it for future use - p.imports[importPath] = pkg - return pkg, nil -} - -func (p *parameterParser) findType(pkg *ast.Package, typeName string) (*ast.TypeSpec, *ast.File, error) { - var file *ast.File - var found *ast.TypeSpec - for _, f := range pkg.Files { - ast.Inspect(f, func(node ast.Node) bool { - ts, ok := node.(*ast.TypeSpec) - if !ok { - return true - } - if ts.Name.Name != typeName { - return true - } - - // found our type, store the file and type - file = f - found = ts - return false - }) - if found != nil { - // already found the type - break - } - } - if found == nil { - return nil, nil, fmt.Errorf("could not find type %v in package %v", typeName, pkg.Name) - } - return found, file, nil -} - -func (p *parameterParser) findImportSpec(se *ast.SelectorExpr) (*ast.ImportSpec, error) { - impName := se.X.(*ast.Ident).Name - for _, i := range p.file.Imports { - if (i.Name != nil && i.Name.Name == impName) || - strings.HasSuffix(strings.Trim(i.Path.Value, `"`), impName) { - return i, nil - } - } - return nil, fmt.Errorf("could not find import %q", impName) -} - -func (p *parameterParser) attachPrefix(f *ast.Field, params map[string]sdk.Parameter) map[string]sdk.Parameter { - // attach prefix if a tag is present or if the field is named - prefix := p.getTag(f.Tag, tagParamName) - if prefix == "" && len(f.Names) > 0 { - prefix = p.formatFieldName(f.Names[0].Name) - } - if prefix == "" { - // no prefix to attach - return params - } - - prefixedParams := make(map[string]sdk.Parameter) - for k, v := range params { - prefixedParams[prefix+fieldSeparator+k] = v - } - return prefixedParams -} - -func (p *parameterParser) isBuiltinType(name string) bool { - switch name { - case "string", "bool", "int", "uint", "int8", "uint8", "int16", "uint16", "int32", "uint32", "int64", "uint64", - "byte", "rune", "float32", "float64": - return true - default: - return false - } -} - -func (p *parameterParser) parseSingleParameter(f *ast.Field, t sdk.ParameterType) (paramName string, param sdk.Parameter, err error) { - fieldName, err := p.getFieldName(f) - if err != nil { - return "", sdk.Parameter{}, err - } - - paramName = p.getTag(f.Tag, tagParamName) - if paramName == "" { - // if there's no tag use the formatted field paramName - paramName = p.formatFieldName(fieldName) - } - - var validations []sdk.Validation - validate := p.getTag(f.Tag, tagParamValidate) - if validate != "" { - validations, err = p.parseValidateTag(validate) - if err != nil { - return "", sdk.Parameter{}, err - } - } - - return paramName, sdk.Parameter{ - Default: p.getTag(f.Tag, tagParamDefault), - Description: p.formatFieldComment(f, fieldName, paramName), - Validations: validations, - Type: t, - }, nil -} - -func (p *parameterParser) getFieldName(f *ast.Field) (string, error) { - if len(f.Names) == 1 { - return f.Names[0].Name, nil - } - - switch v := f.Type.(type) { - case *ast.Ident: - return v.Name, nil - case *ast.SelectorExpr: - return v.Sel.Name, nil - default: - return "", fmt.Errorf("unexpected type: %T", f.Type) - } -} - -func (p *parameterParser) getParamType(i *ast.Ident) sdk.ParameterType { - switch i.Name { - case "int8", "uint8", "int16", "uint16", "int32", "rune", "uint32", "int64", "uint64", "int", "uint": - return sdk.ParameterTypeInt - case "float32", "float64": - return sdk.ParameterTypeFloat - case "bool": - return sdk.ParameterTypeBool - default: - return sdk.ParameterTypeString - } -} - -// formatFieldName formats the name to a camel case string that starts with a -// lowercase letter. If the string starts with multiple uppercase letters, all -// but the last character in the sequence will be converted into lowercase -// letters (e.g. HTTPRequest -> httpRequest). -func (p *parameterParser) formatFieldName(name string) string { - if name == "" { - return "" - } - nameRunes := []rune(name) - foundLowercase := false - i := 0 - newName := strings.Map(func(r rune) rune { - if foundLowercase { - return r - } - if unicode.IsLower(r) { - // short circuit - foundLowercase = true - return r - } - if i == 0 || - (len(nameRunes) > i+1 && unicode.IsUpper(nameRunes[i+1])) { - r = unicode.ToLower(r) - } - i++ - return r - }, name) - return newName -} - -func (p *parameterParser) formatFieldComment(f *ast.Field, fieldName, paramName string) string { - doc := f.Doc - if doc == nil { - // fallback to line comment - doc = f.Comment - } - c := strings.ReplaceAll(doc.Text(), fieldName, paramName) - if len(c) == 0 { - return c - } - - whitespacePrefix := "" - for _, r := range c { - if !unicode.IsSpace(r) { - break - } - whitespacePrefix += string(r) - } - - // get rid of whitespace in first line - c = strings.TrimPrefix(c, whitespacePrefix) - // get rid of whitespace in front of all other lines - c = strings.ReplaceAll(c, "\n"+whitespacePrefix, "\n") - // get rid of new lines and use a space instead - c = strings.ReplaceAll(c, "\n", " ") - // trim space (get rid of any eventual new lines at the end) - c = strings.Trim(c, " ") - return c -} - -func (p *parameterParser) getTag(lit *ast.BasicLit, tag string) string { - if lit == nil { - return "" - } - - st := reflect.StructTag(strings.Trim(lit.Value, "`")) - return st.Get(tag) -} - -func (p *parameterParser) parseValidateTag(tag string) ([]sdk.Validation, error) { - validations := make([]sdk.Validation, 0) - split := strings.Split(tag, tagSeparator) - - for i, s := range split { - s = strings.TrimSpace(s) - split[i] = s - v, err := p.parseValidation(split[i]) - if err != nil { - return nil, err - } - if v != nil { - validations = append(validations, v) - } - } - return validations, nil -} - -func (p *parameterParser) parseValidation(str string) (sdk.Validation, error) { - if str == validationRequired { - return sdk.ValidationRequired{}, nil - } - split := strings.Split(str, validateSeparator) - if len(split) != 2 { - return nil, fmt.Errorf("invalid tag format") - } - - switch split[0] { - case validationRequired: - req, err := strconv.ParseBool(split[1]) - if err != nil { - return nil, err - } - // if required=false then do not add a validation - if !req { - return nil, nil - } - return sdk.ValidationRequired{}, nil - case validationLT, validationLessThan: - val, err := strconv.ParseFloat(split[1], 64) - if err != nil { - return nil, err - } - return sdk.ValidationLessThan{Value: val}, nil - case validationGT, validationGreaterThan: - val, err := strconv.ParseFloat(split[1], 64) - if err != nil { - return nil, err - } - return sdk.ValidationGreaterThan{Value: val}, nil - case validationInclusion: - list := strings.Split(split[1], listSeparator) - return sdk.ValidationInclusion{List: list}, nil - case validationExclusion: - list := strings.Split(split[1], listSeparator) - return sdk.ValidationExclusion{List: list}, nil - case validationRegex: - return sdk.ValidationRegex{Regex: regexp.MustCompile(split[1])}, nil - default: - return nil, fmt.Errorf("invalid value for tag validate: %s", str) - } -} diff --git a/cmd/paramgen/internal/paramgen_test.go b/cmd/paramgen/internal/paramgen_test.go deleted file mode 100644 index fa2b2573..00000000 --- a/cmd/paramgen/internal/paramgen_test.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "errors" - "regexp" - "testing" - - sdk "github.com/conduitio/conduit-connector-sdk" - "github.com/matryer/is" -) - -func TestParseSpecificationSuccess(t *testing.T) { - testCases := []struct { - path string - name string - pkg string - want map[string]sdk.Parameter - }{{ - path: "./testdata/basic", - name: "SourceConfig", - pkg: "example", - want: map[string]sdk.Parameter{ - "foo": { - Default: "bar", - Description: "foo is a required field in the global config with the name \"foo\" and default value \"bar\".", - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - }, - }, - "myString": { - Description: "myString my string description", - Type: sdk.ParameterTypeString, - }, - "myBool": {Type: sdk.ParameterTypeBool}, - "myInt": { - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{ - sdk.ValidationLessThan{ - Value: 100, - }, - sdk.ValidationGreaterThan{ - Value: 0, - }, - }, - }, - "myUint": {Type: sdk.ParameterTypeInt}, - "myInt8": {Type: sdk.ParameterTypeInt}, - "myUint8": {Type: sdk.ParameterTypeInt}, - "myInt16": {Type: sdk.ParameterTypeInt}, - "myUint16": {Type: sdk.ParameterTypeInt}, - "myInt32": {Type: sdk.ParameterTypeInt}, - "myUint32": {Type: sdk.ParameterTypeInt}, - "myInt64": {Type: sdk.ParameterTypeInt}, - "myUint64": {Type: sdk.ParameterTypeInt}, - "myByte": {Type: sdk.ParameterTypeString}, - "myRune": {Type: sdk.ParameterTypeInt}, - "myFloat32": {Type: sdk.ParameterTypeFloat}, - "myFloat64": {Type: sdk.ParameterTypeFloat}, - "myDuration": {Type: sdk.ParameterTypeDuration}, - "myIntSlice": {Type: sdk.ParameterTypeString}, - "myFloatSlice": {Type: sdk.ParameterTypeString}, - "myDurSlice": {Type: sdk.ParameterTypeString}, - "myStringMap.*": {Type: sdk.ParameterTypeString}, - "myStructMap.*.myInt": {Type: sdk.ParameterTypeInt}, - "myStructMap.*.myString": {Type: sdk.ParameterTypeString}, - }, - }, - { - path: "./testdata/complex", - name: "SourceConfig", - pkg: "example", - want: map[string]sdk.Parameter{ - "global.duration": { - Default: "1s", - Description: "duration does not have a name so the type name is used.", - Type: sdk.ParameterTypeDuration, - }, - "global.wildcardStrings.*": { - Default: "foo", - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - }, - }, - "global.renamed.*": { - Default: "1s", - Type: sdk.ParameterTypeDuration, - }, - "global.wildcardStructs.*.name": { - Type: sdk.ParameterTypeString, - }, - "nestMeHere.anotherNested": { - Type: sdk.ParameterTypeInt, - Description: "nestMeHere.anotherNested is also nested under nestMeHere. This is a block comment.", - }, - "nestMeHere.formatThisName": { - Type: sdk.ParameterTypeFloat, - Default: "this is not a float", - Description: "formatThisName should become \"formatThisName\". Default is not a float but that's not a problem, paramgen does not validate correctness.", - }, - "customType": { - Type: sdk.ParameterTypeDuration, - Description: "customType uses a custom type that is convertible to a supported type. Line comments are allowed.", - }, - }, - }, - { - path: "./testdata/tags", - name: "Config", - pkg: "tags", - want: map[string]sdk.Parameter{ - "my-name": { - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{sdk.ValidationRequired{}}, - }, - "my-param": { - Type: sdk.ParameterTypeInt, - Description: "my-param i am a parameter comment", - Default: "3", - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - sdk.ValidationGreaterThan{Value: 0}, - sdk.ValidationLessThan{Value: 100}, - }, - }, - "param2": { - Type: sdk.ParameterTypeBool, - Default: "t", - Validations: []sdk.Validation{ - sdk.ValidationInclusion{List: []string{"true", "t"}}, - sdk.ValidationExclusion{List: []string{"false", "f"}}, - }, - }, - "param3": { - Type: sdk.ParameterTypeString, - Default: "yes", - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - sdk.ValidationRegex{Regex: regexp.MustCompile(".*")}, - }, - }, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.path, func(t *testing.T) { - is := is.New(t) - got, pkg, err := ParseParameters(tc.path, tc.name) - is.NoErr(err) - is.Equal(pkg, tc.pkg) - is.Equal(got, tc.want) - }) - } -} - -func TestParseSpecificationFail(t *testing.T) { - testCases := []struct { - path string - name string - wantErr error - }{{ - path: "./testdata/invalid1", - name: "SourceConfig", - wantErr: errors.New("we do not support parameters from package net/http (please use builtin types or time.Duration)"), - }, { - path: "./testdata/invalid2", - name: "SourceConfig", - wantErr: errors.New("invalid value for tag validate: invalidValidation=hi"), - }, { - path: "./testdata/basic", - name: "SomeConfig", - wantErr: errors.New("struct \"SomeConfig\" was not found in the package \"example\""), - }} - - for _, tc := range testCases { - t.Run(tc.path, func(t *testing.T) { - is := is.New(t) - _, pkg, err := ParseParameters(tc.path, tc.name) - is.Equal(pkg, "") - is.True(err != nil) - for { - unwrapped := errors.Unwrap(err) - if unwrapped == nil { - break - } - err = unwrapped - } - is.Equal(err, tc.wantErr) - }) - } -} diff --git a/cmd/paramgen/internal/template.go b/cmd/paramgen/internal/template.go deleted file mode 100644 index 974286b3..00000000 --- a/cmd/paramgen/internal/template.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "bytes" - "fmt" - "go/format" - "log" - "reflect" - "strconv" - "text/template" - - sdk "github.com/conduitio/conduit-connector-sdk" -) - -const ( - tmpl = `// Code generated by paramgen. DO NOT EDIT. -// Source: github.com/ConduitIO/conduit-connector-sdk/tree/main/cmd/paramgen - -package {{ $.Package }} - -import ( - {{- if $.HasRegex }} - "regexp" - {{ end }} - sdk "github.com/conduitio/conduit-connector-sdk" -) - -func ({{ $.Struct }}) Parameters() map[string]sdk.Parameter { - return map[string]sdk.Parameter{ - {{- range $name, $parameter := .Parameters }} - {{ $.Quote $name }}: { - Default: {{ $.Quote .Default }}, - Description: {{ $.Quote .Description }}, - Type: sdk.{{ .GetTypeConstant }}, - Validations: []sdk.Validation{ - {{- range $index, $validation := .Validations }} - {{ $parameter.GetValidation $index }}, - {{- end }} - }, - }, - {{- end }} - } -} -` -) - -type templateData struct { - Parameters map[string]parameter - Package string - Struct string -} - -func (templateData) Quote(s string) string { - return strconv.Quote(s) -} - -var parameterTypeConstantMapping = map[sdk.ParameterType]string{ - sdk.ParameterTypeString: "ParameterTypeString", - sdk.ParameterTypeInt: "ParameterTypeInt", - sdk.ParameterTypeFloat: "ParameterTypeFloat", - sdk.ParameterTypeBool: "ParameterTypeBool", - sdk.ParameterTypeFile: "ParameterTypeFile", - sdk.ParameterTypeDuration: "ParameterTypeDuration", -} - -type parameter sdk.Parameter - -func (p parameter) GetTypeConstant() string { - return parameterTypeConstantMapping[p.Type] -} -func (p parameter) GetValidation(index int) string { - validation := p.Validations[index] - - regexValidation, ok := validation.(sdk.ValidationRegex) - if !ok { - // default behavior - return fmt.Sprintf("%#v", p.Validations[index]) - } - - validationType := reflect.TypeOf(validation).String() - validationParameters := fmt.Sprintf("Regex: regexp.MustCompile(%q)", regexValidation.Regex) - return fmt.Sprintf("%s{%s}", validationType, validationParameters) -} - -func (t templateData) HasRegex() bool { - for _, p := range t.Parameters { - for _, v := range p.Validations { - if _, ok := v.(sdk.ValidationRegex); ok { - return true - } - } - } - return false -} - -func GenerateCode(parameters map[string]sdk.Parameter, packageName string, structName string) string { - // create the go template - t := template.Must(template.New("").Parse(tmpl)) - - internalParams := make(map[string]parameter, len(parameters)) - for k, v := range parameters { - internalParams[k] = parameter(v) - } - - data := templateData{ - Package: packageName, - Struct: structName, - Parameters: internalParams, - } - var processed bytes.Buffer - // execute the template - err := t.Execute(&processed, data) - if err != nil { - log.Fatalf("error: failed to execute template: %v", err) - } - - // format the output as Go code in the “gofmt” style - formatted, err := format.Source(processed.Bytes()) - if err != nil { - log.Fatalf("error: could not format processed template: %v", err) - } - - return string(formatted) -} diff --git a/cmd/paramgen/internal/template_test.go b/cmd/paramgen/internal/template_test.go deleted file mode 100644 index 192251d0..00000000 --- a/cmd/paramgen/internal/template_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "regexp" - "testing" - - sdk "github.com/conduitio/conduit-connector-sdk" - "github.com/matryer/is" -) - -func TestGenerateCodeWithRegex(t *testing.T) { - is := is.New(t) - got := GenerateCode(map[string]sdk.Parameter{ - "int.param": { - Default: "1", - Description: "my int param with \"quotes\"", - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - sdk.ValidationExclusion{List: []string{"0", "-1"}}, - sdk.ValidationGreaterThan{Value: -3}, - sdk.ValidationLessThan{Value: 3}, - }, - }, - "bool.param": { - Default: "true", - Description: "my bool param", - Type: sdk.ParameterTypeBool, - Validations: []sdk.Validation{ - sdk.ValidationRegex{Regex: regexp.MustCompile(".*")}, - }, - }, - "string.param": { - Description: "simple string param", - Type: sdk.ParameterTypeString, - }, - }, "s3", "SourceConfig") - - want := `// Code generated by paramgen. DO NOT EDIT. -// Source: github.com/ConduitIO/conduit-connector-sdk/tree/main/cmd/paramgen - -package s3 - -import ( - "regexp" - - sdk "github.com/conduitio/conduit-connector-sdk" -) - -func (SourceConfig) Parameters() map[string]sdk.Parameter { - return map[string]sdk.Parameter{ - "bool.param": { - Default: "true", - Description: "my bool param", - Type: sdk.ParameterTypeBool, - Validations: []sdk.Validation{ - sdk.ValidationRegex{Regex: regexp.MustCompile(".*")}, - }, - }, - "int.param": { - Default: "1", - Description: "my int param with \"quotes\"", - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{ - sdk.ValidationRequired{}, - sdk.ValidationExclusion{List: []string{"0", "-1"}}, - sdk.ValidationGreaterThan{Value: -3}, - sdk.ValidationLessThan{Value: 3}, - }, - }, - "string.param": { - Default: "", - Description: "simple string param", - Type: sdk.ParameterTypeString, - Validations: []sdk.Validation{}, - }, - } -} -` - is.Equal(got, want) -} - -func TestGenerateCodeWithoutRegex(t *testing.T) { - is := is.New(t) - got := GenerateCode(map[string]sdk.Parameter{ - "int.param": { - Default: "1", - Description: "my int param", - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{}, - }, - "duration.param": { - Default: "1s", - Description: "my duration param", - Type: sdk.ParameterTypeDuration, - Validations: []sdk.Validation{ - sdk.ValidationInclusion{List: []string{"1s", "2s", "3s"}}, - }, - }, - }, "file", "Config") - - want := `// Code generated by paramgen. DO NOT EDIT. -// Source: github.com/ConduitIO/conduit-connector-sdk/tree/main/cmd/paramgen - -package file - -import ( - sdk "github.com/conduitio/conduit-connector-sdk" -) - -func (Config) Parameters() map[string]sdk.Parameter { - return map[string]sdk.Parameter{ - "duration.param": { - Default: "1s", - Description: "my duration param", - Type: sdk.ParameterTypeDuration, - Validations: []sdk.Validation{ - sdk.ValidationInclusion{List: []string{"1s", "2s", "3s"}}, - }, - }, - "int.param": { - Default: "1", - Description: "my int param", - Type: sdk.ParameterTypeInt, - Validations: []sdk.Validation{}, - }, - } -} -` - is.Equal(got, want) -} diff --git a/cmd/paramgen/internal/testdata/basic/go.mod b/cmd/paramgen/internal/testdata/basic/go.mod deleted file mode 100644 index 04fcff98..00000000 --- a/cmd/paramgen/internal/testdata/basic/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/test - -go 1.18 diff --git a/cmd/paramgen/internal/testdata/basic/specs.go b/cmd/paramgen/internal/testdata/basic/specs.go deleted file mode 100644 index efed368d..00000000 --- a/cmd/paramgen/internal/testdata/basic/specs.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package example - -import ( - "net/http" - "time" -) - -// GlobalConfig is a reusable config struct used in the source and destination -// config. -type GlobalConfig struct { - // MyGlobalString is a required field in the global config with the name - // "foo" and default value "bar". - MyGlobalString string `json:"foo" default:"bar" validate:"required"` -} - -// SourceConfig this comment will be ignored. -type SourceConfig struct { - GlobalConfig - - // MyString my string description - MyString string - MyBool bool - - MyInt int `validate:"lt=100, gt=0"` - MyUint uint - MyInt8 int8 - MyUint8 uint8 - MyInt16 int16 - MyUint16 uint16 - MyInt32 int32 - MyUint32 uint32 - MyInt64 int64 - MyUint64 uint64 - - MyByte byte - MyRune rune - - MyFloat32 float32 - MyFloat64 float64 - - MyDuration time.Duration - - MyIntSlice []int - MyFloatSlice []float32 - MyDurSlice []time.Duration - - MyStringMap map[string]string - MyStructMap map[string]structMapVal - - // this field is ignored because it is not exported - ignoreThis http.Client -} - -type structMapVal struct { - MyString string - MyInt int -} diff --git a/cmd/paramgen/internal/testdata/complex/global.go b/cmd/paramgen/internal/testdata/complex/global.go deleted file mode 100644 index e0c6b9e7..00000000 --- a/cmd/paramgen/internal/testdata/complex/global.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// go:build ignoreBuildTags - -package example - -import ( - "example.com/test/internal" -) - -// GlobalConfig is defined in another file. It is defined with an underlying -// type that is in another package (should still work). -type GlobalConfig internal.GlobalConfig diff --git a/cmd/paramgen/internal/testdata/complex/go.mod b/cmd/paramgen/internal/testdata/complex/go.mod deleted file mode 100644 index 04fcff98..00000000 --- a/cmd/paramgen/internal/testdata/complex/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/test - -go 1.18 diff --git a/cmd/paramgen/internal/testdata/complex/internal/global.go b/cmd/paramgen/internal/testdata/complex/internal/global.go deleted file mode 100644 index c51b5c8e..00000000 --- a/cmd/paramgen/internal/testdata/complex/internal/global.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import "time" - -// GlobalConfig is an internal struct that paramgen still parses. -type GlobalConfig struct { - // Duration does not have a name so the type name is used. - time.Duration `default:"1s"` // line comments on fields with doc comments are ignored - - WildcardStrings map[string]string `default:"foo" validate:"required"` - WildcardInts map[string]time.Duration `json:"renamed" default:"1s"` - WildcardStructs WildcardStruct -} - -type WildcardStruct map[string]struct { - Name string -} diff --git a/cmd/paramgen/internal/testdata/complex/specs.go b/cmd/paramgen/internal/testdata/complex/specs.go deleted file mode 100644 index 87fb85e4..00000000 --- a/cmd/paramgen/internal/testdata/complex/specs.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package example - -import "time" - -type SourceConfig struct { - // GlobalConfig parameters should be nested under "global". This comment - // should be ignored. - Global GlobalConfig `json:"global"` - // Nested structs can be used to create namespaces - Nested struct { - // FORMATThisName should become "formatThisName". Default is not a float - // but that's not a problem, paramgen does not validate correctness. - FORMATThisName float32 `default:"this is not a float"` - // unexported fields should be ignored. - unexportedField string - } `json:"nestMeHere"` - /* - AnotherNested is also nested under nestMeHere. - This is a block comment. - */ - AnotherNested int `json:"nestMeHere.anotherNested"` - CustomType CustomDuration // CustomType uses a custom type that is convertible to a supported type. Line comments are allowed. -} - -type CustomDuration CustomDuration2 -type CustomDuration2 time.Duration diff --git a/cmd/paramgen/internal/testdata/complex/specs_test.go b/cmd/paramgen/internal/testdata/complex/specs_test.go deleted file mode 100644 index 9bdabb55..00000000 --- a/cmd/paramgen/internal/testdata/complex/specs_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package example_test - -import ( - "testing" -) - -func TestConfig(t *testing.T) { - tests are not parsed at all by paramgen so this file can contain code that doesn't compile -} diff --git a/cmd/paramgen/internal/testdata/complex/tools.go b/cmd/paramgen/internal/testdata/complex/tools.go deleted file mode 100644 index 7360cec5..00000000 --- a/cmd/paramgen/internal/testdata/complex/tools.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright © 2024 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build tools - -// This file should be ignored because of the build tag. - -package main diff --git a/cmd/paramgen/internal/testdata/invalid1/go.mod b/cmd/paramgen/internal/testdata/invalid1/go.mod deleted file mode 100644 index 04fcff98..00000000 --- a/cmd/paramgen/internal/testdata/invalid1/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/test - -go 1.18 diff --git a/cmd/paramgen/internal/testdata/invalid1/specs.go b/cmd/paramgen/internal/testdata/invalid1/specs.go deleted file mode 100644 index f580535f..00000000 --- a/cmd/paramgen/internal/testdata/invalid1/specs.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package example - -import "net/http" - -type SourceConfig struct { - // We don't support types imported from packages outside this module. - InvalidType http.Client -} diff --git a/cmd/paramgen/internal/testdata/invalid2/go.mod b/cmd/paramgen/internal/testdata/invalid2/go.mod deleted file mode 100644 index 04fcff98..00000000 --- a/cmd/paramgen/internal/testdata/invalid2/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/test - -go 1.18 diff --git a/cmd/paramgen/internal/testdata/invalid2/specs.go b/cmd/paramgen/internal/testdata/invalid2/specs.go deleted file mode 100644 index c9e0f282..00000000 --- a/cmd/paramgen/internal/testdata/invalid2/specs.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package example - -type SourceConfig struct { - MyInt int `validate:"invalidValidation=hi"` -} diff --git a/cmd/paramgen/internal/testdata/tags/go.mod b/cmd/paramgen/internal/testdata/tags/go.mod deleted file mode 100644 index 04fcff98..00000000 --- a/cmd/paramgen/internal/testdata/tags/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/test - -go 1.18 diff --git a/cmd/paramgen/internal/testdata/tags/specs.go b/cmd/paramgen/internal/testdata/tags/specs.go deleted file mode 100644 index 93689694..00000000 --- a/cmd/paramgen/internal/testdata/tags/specs.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tags - -// Config is a reusable config struct used in the source and destination -type Config struct { - InnerConfig - // Param1 i am a parameter comment - Param1 int `validate:"required, gt=0, lt=100" default:"3" json:"my-param"` - - Param2 bool `validate:"inclusion=true|t, exclusion=false|f, required=false" default:"t"` - Param3 string `validate:"required=true, regex=.*" default:"yes"` -} - -type InnerConfig struct { - Name string `validate:"required" json:"my-name"` -} diff --git a/cmd/paramgen/main.go b/cmd/paramgen/main.go deleted file mode 100644 index d8a7bae0..00000000 --- a/cmd/paramgen/main.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright © 2023 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "log" - "os" - "strings" - - "github.com/conduitio/conduit-connector-sdk/cmd/paramgen/internal" -) - -func main() { - log.SetFlags(0) - log.SetPrefix("paramgen: ") - - // parse the command arguments - args := parseFlags() - - // parse the sdk parameters - params, pkg, err := internal.ParseParameters(args.path, args.structName) - if err != nil { - log.Fatalf("error: failed to parse parameters: %v", err) - } - - code := internal.GenerateCode(params, pkg, args.structName) - - path := strings.TrimSuffix(args.path, "/") + "/" + args.output - err = os.WriteFile(path, []byte(code), 0644) - if err != nil { - log.Fatalf("error: failed to output file: %v", err) - } -} - -type Args struct { - output string - path string - structName string -} - -func parseFlags() Args { - flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - var ( - output = flags.String("output", "paramgen.go", "name of the output file") - path = flags.String("path", ".", "directory path to the package that has the configuration struct") - ) - - // flags is set up to exit on error, we can safely ignore the error - _ = flags.Parse(os.Args[1:]) - - if len(flags.Args()) == 0 { - log.Println("error: struct name argument missing") - fmt.Println() - flags.Usage() - os.Exit(1) - } - - var args Args - args.output = stringPtrToVal(output) - args.path = stringPtrToVal(path) - args.structName = flags.Args()[0] - - // add .go suffix if it is not in the name - if !strings.HasSuffix(args.output, ".go") { - args.output += ".go" - } - - return args -} - -func stringPtrToVal(s *string) string { - if s == nil { - return "" - } - return *s -} diff --git a/destination.go b/destination.go index a3d1ca11..64de47f2 100644 --- a/destination.go +++ b/destination.go @@ -26,7 +26,8 @@ import ( "time" "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" + "github.com/conduitio/conduit-commons/opencdc" + "github.com/conduitio/conduit-connector-protocol/cplugin" "github.com/conduitio/conduit-connector-sdk/internal" "github.com/conduitio/conduit-connector-sdk/internal/csync" ) @@ -38,7 +39,7 @@ import ( type Destination interface { // Parameters is a map of named Parameters that describe how to configure // the Destination. - Parameters() map[string]Parameter + Parameters() config.Parameters // Configure is the first function to be called in a connector. It provides the // connector with the configuration that needs to be validated and stored. @@ -61,7 +62,7 @@ type Destination interface { // caching. It should return the number of records written from r // (0 <= n <= len(r)) and any error encountered that caused the write to // stop early. Write must return a non-nil error if it returns n < len(r). - Write(ctx context.Context, r []Record) (n int, err error) + Write(ctx context.Context, r []opencdc.Record) (n int, err error) // Teardown signals to the plugin that all records were written and there // will be no more calls to any other function. After Teardown returns, the @@ -93,9 +94,9 @@ type Destination interface { } // NewDestinationPlugin takes a Destination and wraps it into an adapter that -// converts it into a cpluginv1.DestinationPlugin. If the parameter is nil it +// converts it into a cplugin.DestinationPlugin. If the parameter is nil it // will wrap UnimplementedDestination instead. -func NewDestinationPlugin(impl Destination) cpluginv1.DestinationPlugin { +func NewDestinationPlugin(impl Destination) cplugin.DestinationPlugin { if impl == nil { // prevent nil pointers impl = UnimplementedDestination{} @@ -106,18 +107,19 @@ func NewDestinationPlugin(impl Destination) cpluginv1.DestinationPlugin { type destinationPluginAdapter struct { impl Destination - lastPosition *csync.ValueWatcher[Position] + lastPosition *csync.ValueWatcher[opencdc.Position] openCancel context.CancelFunc // write is the chosen write strategy, either single records or batches writeStrategy writeStrategy } -func (a *destinationPluginAdapter) Configure(ctx context.Context, req cpluginv1.DestinationConfigureRequest) (cpluginv1.DestinationConfigureResponse, error) { +func (a *destinationPluginAdapter) Configure(ctx context.Context, req cplugin.DestinationConfigureRequest) (cplugin.DestinationConfigureResponse, error) { ctx = DestinationWithBatch{}.setBatchEnabled(ctx, false) - params := parameters(a.impl.Parameters()).toConfigParameters() + params := a.impl.Parameters() + // TODO should we stop doing this here? The Processor SDK does NOT do this. // sanitize config and apply default values cfg := config.Config(req.Config). Sanitize(). @@ -131,7 +133,7 @@ func (a *destinationPluginAdapter) Configure(ctx context.Context, req cpluginv1. // configure write strategy errs = append(errs, a.configureWriteStrategy(ctx, cfg)) - return cpluginv1.DestinationConfigureResponse{}, errors.Join(errs...) + return cplugin.DestinationConfigureResponse{}, errors.Join(errs...) } func (a *destinationPluginAdapter) configureWriteStrategy(ctx context.Context, config map[string]string) error { @@ -178,8 +180,8 @@ func (a *destinationPluginAdapter) configureWriteStrategy(ctx context.Context, c return nil } -func (a *destinationPluginAdapter) Start(ctx context.Context, _ cpluginv1.DestinationStartRequest) (cpluginv1.DestinationStartResponse, error) { - a.lastPosition = new(csync.ValueWatcher[Position]) +func (a *destinationPluginAdapter) Start(ctx context.Context, _ cplugin.DestinationStartRequest) (cplugin.DestinationStartResponse, error) { + a.lastPosition = new(csync.ValueWatcher[opencdc.Position]) // detach context, so we can control when it's canceled ctxOpen := internal.DetachContext(ctx) @@ -200,12 +202,12 @@ func (a *destinationPluginAdapter) Start(ctx context.Context, _ cpluginv1.Destin }() err := a.impl.Open(ctxOpen) - return cpluginv1.DestinationStartResponse{}, err + return cplugin.DestinationStartResponse{}, err } -func (a *destinationPluginAdapter) Run(ctx context.Context, stream cpluginv1.DestinationRunStream) error { - for { - req, err := stream.Recv() +func (a *destinationPluginAdapter) Run(ctx context.Context, stream cplugin.DestinationRunStream) error { + for stream := stream.Server(); ; { + batch, err := stream.Recv() if err != nil { if err == io.EOF { // stream is closed @@ -213,27 +215,30 @@ func (a *destinationPluginAdapter) Run(ctx context.Context, stream cpluginv1.Des } return fmt.Errorf("write stream error: %w", err) } - r := a.convertRecord(req.Record) - err = a.writeStrategy.Write(ctx, r, func(err error) error { - return a.ack(r, err, stream) - }) - a.lastPosition.Set(r.Position) - if err != nil { - return err + for _, rec := range batch.Records { + err = a.writeStrategy.Write(ctx, rec, func(err error) error { + return a.ack(rec, err, stream) + }) + a.lastPosition.Set(rec.Position) + if err != nil { + return err + } } } } // ack sends a message into the stream signaling that the record was processed. -func (a *destinationPluginAdapter) ack(r Record, writeErr error, stream cpluginv1.DestinationRunStream) error { +func (a *destinationPluginAdapter) ack(r opencdc.Record, writeErr error, stream cplugin.DestinationRunStreamServer) error { var ackErrStr string if writeErr != nil { ackErrStr = writeErr.Error() } - err := stream.Send(cpluginv1.DestinationRunResponse{ - AckPosition: r.Position, - Error: ackErrStr, + err := stream.Send(cplugin.DestinationRunResponse{ + Acks: []cplugin.DestinationRunResponseAck{{ + Position: r.Position, + Error: ackErrStr, + }}, }) if err != nil { return fmt.Errorf("ack stream error: %w", err) @@ -249,7 +254,7 @@ func (a *destinationPluginAdapter) ack(r Record, writeErr error, stream cpluginv // flushing records received so far and return an error. Flushing of records // also has a timeout of 1 minute, after which the stop operation returns with // an error. In the worst case this operation can thus take 2 minutes. -func (a *destinationPluginAdapter) Stop(ctx context.Context, req cpluginv1.DestinationStopRequest) (cpluginv1.DestinationStopResponse, error) { +func (a *destinationPluginAdapter) Stop(ctx context.Context, req cplugin.DestinationStopRequest) (cplugin.DestinationStopResponse, error) { // last thing we do is cancel context in Open defer a.openCancel() @@ -257,7 +262,7 @@ func (a *destinationPluginAdapter) Stop(ctx context.Context, req cpluginv1.Desti // to flush what we have so far actualLastPosition, err := a.lastPosition.Watch( ctx, - func(val Position) bool { + func(val opencdc.Position) bool { return bytes.Equal(val, req.LastPosition) }, csync.WithTimeout(stopTimeout), @@ -278,10 +283,10 @@ func (a *destinationPluginAdapter) Stop(ctx context.Context, req cpluginv1.Desti Logger(ctx).Err(err).Msg("error flushing records") } - return cpluginv1.DestinationStopResponse{}, err + return cplugin.DestinationStopResponse{}, err } -func (a *destinationPluginAdapter) Teardown(ctx context.Context, _ cpluginv1.DestinationTeardownRequest) (cpluginv1.DestinationTeardownResponse, error) { +func (a *destinationPluginAdapter) Teardown(ctx context.Context, _ cplugin.DestinationTeardownRequest) (cplugin.DestinationTeardownResponse, error) { // cancel open context, in case Stop was not called (can happen in case the // stop was triggered by an error) // teardown can be called without "open" being called previously @@ -293,66 +298,25 @@ func (a *destinationPluginAdapter) Teardown(ctx context.Context, _ cpluginv1.Des err := a.impl.Teardown(ctx) if err != nil { - return cpluginv1.DestinationTeardownResponse{}, err - } - return cpluginv1.DestinationTeardownResponse{}, nil -} - -func (a *destinationPluginAdapter) LifecycleOnCreated(ctx context.Context, req cpluginv1.DestinationLifecycleOnCreatedRequest) (cpluginv1.DestinationLifecycleOnCreatedResponse, error) { - return cpluginv1.DestinationLifecycleOnCreatedResponse{}, a.impl.LifecycleOnCreated(ctx, req.Config) -} -func (a *destinationPluginAdapter) LifecycleOnUpdated(ctx context.Context, req cpluginv1.DestinationLifecycleOnUpdatedRequest) (cpluginv1.DestinationLifecycleOnUpdatedResponse, error) { - return cpluginv1.DestinationLifecycleOnUpdatedResponse{}, a.impl.LifecycleOnUpdated(ctx, req.ConfigBefore, req.ConfigAfter) -} -func (a *destinationPluginAdapter) LifecycleOnDeleted(ctx context.Context, req cpluginv1.DestinationLifecycleOnDeletedRequest) (cpluginv1.DestinationLifecycleOnDeletedResponse, error) { - return cpluginv1.DestinationLifecycleOnDeletedResponse{}, a.impl.LifecycleOnDeleted(ctx, req.Config) -} - -func (a *destinationPluginAdapter) convertRecord(r cpluginv1.Record) Record { - return Record{ - Position: r.Position, - Operation: Operation(r.Operation), - Metadata: a.convertMetadata(r.Metadata), - Key: a.convertData(r.Key), - Payload: a.convertChange(r.Payload), + return cplugin.DestinationTeardownResponse{}, err } + return cplugin.DestinationTeardownResponse{}, nil } -func (a *destinationPluginAdapter) convertMetadata(m map[string]string) Metadata { - metadata := (Metadata)(m) - if metadata == nil { - metadata = make(map[string]string, 1) - } - metadata.SetOpenCDCVersion() - return metadata +func (a *destinationPluginAdapter) LifecycleOnCreated(ctx context.Context, req cplugin.DestinationLifecycleOnCreatedRequest) (cplugin.DestinationLifecycleOnCreatedResponse, error) { + return cplugin.DestinationLifecycleOnCreatedResponse{}, a.impl.LifecycleOnCreated(ctx, req.Config) } - -func (a *destinationPluginAdapter) convertChange(c cpluginv1.Change) Change { - return Change{ - Before: a.convertData(c.Before), - After: a.convertData(c.After), - } +func (a *destinationPluginAdapter) LifecycleOnUpdated(ctx context.Context, req cplugin.DestinationLifecycleOnUpdatedRequest) (cplugin.DestinationLifecycleOnUpdatedResponse, error) { + return cplugin.DestinationLifecycleOnUpdatedResponse{}, a.impl.LifecycleOnUpdated(ctx, req.ConfigBefore, req.ConfigAfter) } - -func (a *destinationPluginAdapter) convertData(d cpluginv1.Data) Data { - if d == nil { - return nil - } - - switch v := d.(type) { - case cpluginv1.RawData: - return RawData(v) - case cpluginv1.StructuredData: - return StructuredData(v) - default: - panic("unknown data type") - } +func (a *destinationPluginAdapter) LifecycleOnDeleted(ctx context.Context, req cplugin.DestinationLifecycleOnDeletedRequest) (cplugin.DestinationLifecycleOnDeletedResponse, error) { + return cplugin.DestinationLifecycleOnDeletedResponse{}, a.impl.LifecycleOnDeleted(ctx, req.Config) } // writeStrategy is used to switch between writing single records and batching // them. type writeStrategy interface { - Write(ctx context.Context, r Record, ack func(error) error) error + Write(ctx context.Context, r opencdc.Record, ack func(error) error) error Flush(ctx context.Context) error } @@ -363,8 +327,8 @@ type writeStrategySingle struct { impl Destination } -func (w *writeStrategySingle) Write(ctx context.Context, r Record, ack func(error) error) error { - _, err := w.impl.Write(ctx, []Record{r}) +func (w *writeStrategySingle) Write(ctx context.Context, r opencdc.Record, ack func(error) error) error { + _, err := w.impl.Write(ctx, []opencdc.Record{r}) if err != nil { Logger(ctx).Err(err).Bytes("record_position", r.Position).Msg("error writing record") } @@ -386,7 +350,7 @@ type writeStrategyBatch struct { type writeBatchItem struct { ctx context.Context - record Record + record opencdc.Record ack func(error) error } @@ -401,7 +365,7 @@ func newWriteStrategyBatch(impl Destination, batchSize int, batchDelay time.Dura } func (w *writeStrategyBatch) writeBatch(batch []writeBatchItem) error { - records := make([]Record, len(batch)) + records := make([]opencdc.Record, len(batch)) for i, item := range batch { records[i] = item.record } @@ -436,7 +400,7 @@ func (w *writeStrategyBatch) writeBatch(batch []writeBatchItem) error { return firstErr } -func (w *writeStrategyBatch) Write(ctx context.Context, r Record, ack func(error) error) error { +func (w *writeStrategyBatch) Write(ctx context.Context, r opencdc.Record, ack func(error) error) error { select { case result := <-w.batcher.Results(): Logger(ctx).Debug(). @@ -513,7 +477,7 @@ type DestinationUtil struct{} // // Example usage: // -// func (d *Destination) Write(ctx context.Context, r sdk.Record) error { +// func (d *Destination) Write(ctx context.Context, r opencdc.Record) error { // return d.Util.Route(ctx, r, // d.handleInsert, // d.handleUpdate, @@ -521,25 +485,25 @@ type DestinationUtil struct{} // d.handleSnapshot, // we could also reuse d.handleInsert // ) // } -// func (d *Destination) handleInsert(ctx context.Context, r sdk.Record) error { +// func (d *Destination) handleInsert(ctx context.Context, r opencdc.Record) error { // ... // } func (DestinationUtil) Route( ctx context.Context, - rec Record, - handleCreate func(context.Context, Record) error, - handleUpdate func(context.Context, Record) error, - handleDelete func(context.Context, Record) error, - handleSnapshot func(context.Context, Record) error, + rec opencdc.Record, + handleCreate func(context.Context, opencdc.Record) error, + handleUpdate func(context.Context, opencdc.Record) error, + handleDelete func(context.Context, opencdc.Record) error, + handleSnapshot func(context.Context, opencdc.Record) error, ) error { switch rec.Operation { - case OperationCreate: + case opencdc.OperationCreate: return handleCreate(ctx, rec) - case OperationUpdate: + case opencdc.OperationUpdate: return handleUpdate(ctx, rec) - case OperationDelete: + case opencdc.OperationDelete: return handleDelete(ctx, rec) - case OperationSnapshot: + case opencdc.OperationSnapshot: return handleSnapshot(ctx, rec) default: return fmt.Errorf("invalid operation %q", rec.Operation) diff --git a/destination_middleware.go b/destination_middleware.go index 858d109f..7031edc3 100644 --- a/destination_middleware.go +++ b/destination_middleware.go @@ -21,6 +21,8 @@ import ( "strconv" "time" + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" "golang.org/x/time/rate" ) @@ -113,17 +115,17 @@ type destinationWithBatch struct { defaults DestinationWithBatch } -func (d *destinationWithBatch) Parameters() map[string]Parameter { - return mergeParameters(d.Destination.Parameters(), map[string]Parameter{ +func (d *destinationWithBatch) Parameters() config.Parameters { + return mergeParameters(d.Destination.Parameters(), config.Parameters{ configDestinationBatchSize: { Default: strconv.Itoa(d.defaults.DefaultBatchSize), Description: "Maximum size of batch before it gets written to the destination.", - Type: ParameterTypeInt, + Type: config.ParameterTypeInt, }, configDestinationBatchDelay: { Default: d.defaults.DefaultBatchDelay.String(), Description: "Maximum delay before an incomplete batch is written to the destination.", - Type: ParameterTypeDuration, + Type: config.ParameterTypeDuration, }, }) } @@ -193,17 +195,17 @@ type destinationWithRateLimit struct { limiter *rate.Limiter } -func (d *destinationWithRateLimit) Parameters() map[string]Parameter { - return mergeParameters(d.Destination.Parameters(), map[string]Parameter{ +func (d *destinationWithRateLimit) Parameters() config.Parameters { + return mergeParameters(d.Destination.Parameters(), config.Parameters{ configDestinationRatePerSecond: { Default: strconv.FormatFloat(d.defaults.DefaultRatePerSecond, 'f', -1, 64), Description: "Maximum times records can be written per second (0 means no rate limit).", - Type: ParameterTypeFloat, + Type: config.ParameterTypeFloat, }, configDestinationRateBurst: { Default: strconv.Itoa(d.defaults.DefaultBurst), Description: "Allow bursts of at most X writes (1 or less means that bursts are not allowed). Only takes effect if a rate limit per second is set.", - Type: ParameterTypeInt, + Type: config.ParameterTypeInt, }, }) } @@ -244,7 +246,7 @@ func (d *destinationWithRateLimit) Configure(ctx context.Context, config map[str return nil } -func (d *destinationWithRateLimit) Write(ctx context.Context, recs []Record) (int, error) { +func (d *destinationWithRateLimit) Write(ctx context.Context, recs []opencdc.Record) (int, error) { if d.limiter != nil { err := d.limiter.Wait(ctx) if err != nil { @@ -270,7 +272,7 @@ const ( type DestinationWithRecordFormat struct { // DefaultRecordFormat is the default record format. DefaultRecordFormat string - RecordFormatters []RecordFormatter + RecordSerializers []RecordSerializer } func (d DestinationWithRecordFormat) RecordFormatParameterName() string { @@ -280,15 +282,15 @@ func (d DestinationWithRecordFormat) RecordFormatOptionsParameterName() string { return configDestinationRecordFormatOptions } -// DefaultRecordFormatters returns the list of record formatters that are used -// if DestinationWithRecordFormat.RecordFormatters is nil. -func (d DestinationWithRecordFormat) DefaultRecordFormatters() []RecordFormatter { - formatters := []RecordFormatter{ - // define specific formatters here - TemplateRecordFormatter{}, +// DefaultRecordSerializers returns the list of record serializers that are used +// if DestinationWithRecordFormat.RecordSerializers is nil. +func (d DestinationWithRecordFormat) DefaultRecordSerializers() []RecordSerializer { + serializers := []RecordSerializer{ + // define specific serializers here + TemplateRecordSerializer{}, } - // add generic formatters here, they are combined in all possible combinations + // add generic serializers here, they are combined in all possible combinations genericConverters := []Converter{ OpenCDCConverter{}, DebeziumConverter{}, @@ -299,29 +301,29 @@ func (d DestinationWithRecordFormat) DefaultRecordFormatters() []RecordFormatter for _, c := range genericConverters { for _, e := range genericEncoders { - formatters = append( - formatters, - GenericRecordFormatter{ + serializers = append( + serializers, + GenericRecordSerializer{ Converter: c, Encoder: e, }, ) } } - return formatters + return serializers } // Wrap a Destination into the record format middleware. func (d DestinationWithRecordFormat) Wrap(impl Destination) Destination { if d.DefaultRecordFormat == "" { - d.DefaultRecordFormat = defaultFormatter.Name() + d.DefaultRecordFormat = defaultSerializer.Name() } - if len(d.RecordFormatters) == 0 { - d.RecordFormatters = d.DefaultRecordFormatters() + if len(d.RecordSerializers) == 0 { + d.RecordSerializers = d.DefaultRecordSerializers() } - // sort record formatters by name to ensure we can binary search them - sort.Slice(d.RecordFormatters, func(i, j int) bool { return d.RecordFormatters[i].Name() < d.RecordFormatters[j].Name() }) + // sort record serializers by name to ensure we can binary search them + sort.Slice(d.RecordSerializers, func(i, j int) bool { return d.RecordSerializers[i].Name() < d.RecordSerializers[j].Name() }) return &destinationWithRecordFormat{ Destination: impl, @@ -333,26 +335,26 @@ type destinationWithRecordFormat struct { Destination defaults DestinationWithRecordFormat - formatter RecordFormatter + serializer RecordSerializer } func (d *destinationWithRecordFormat) formats() []string { - names := make([]string, len(d.defaults.RecordFormatters)) + names := make([]string, len(d.defaults.RecordSerializers)) i := 0 - for _, c := range d.defaults.RecordFormatters { + for _, c := range d.defaults.RecordSerializers { names[i] = c.Name() i++ } return names } -func (d *destinationWithRecordFormat) Parameters() map[string]Parameter { - return mergeParameters(d.Destination.Parameters(), map[string]Parameter{ +func (d *destinationWithRecordFormat) Parameters() config.Parameters { + return mergeParameters(d.Destination.Parameters(), config.Parameters{ configDestinationRecordFormat: { Default: d.defaults.DefaultRecordFormat, Description: "The format of the output record.", - Validations: []Validation{ - ValidationInclusion{List: d.formats()}, + Validations: []config.Validation{ + config.ValidationInclusion{List: d.formats()}, }, }, configDestinationRecordFormatOptions: { @@ -374,23 +376,23 @@ func (d *destinationWithRecordFormat) Configure(ctx context.Context, config map[ i := sort.SearchStrings(d.formats(), format) // if the string is not found i is equal to the size of the slice - if i == len(d.defaults.RecordFormatters) { + if i == len(d.defaults.RecordSerializers) { return fmt.Errorf("invalid %s: %q not found in %v", configDestinationRecordFormat, format, d.formats()) } - formatter := d.defaults.RecordFormatters[i] - formatter, err = formatter.Configure(config[configDestinationRecordFormatOptions]) + serializer := d.defaults.RecordSerializers[i] + serializer, err = serializer.Configure(config[configDestinationRecordFormatOptions]) if err != nil { return fmt.Errorf("invalid %s for %q: %w", configDestinationRecordFormatOptions, format, err) } - d.formatter = formatter + d.serializer = serializer return nil } -func (d *destinationWithRecordFormat) Write(ctx context.Context, recs []Record) (int, error) { +func (d *destinationWithRecordFormat) Write(ctx context.Context, recs []opencdc.Record) (int, error) { for i := range recs { - recs[i].formatter = d.formatter + recs[i].SetSerializer(d.serializer) } return d.Destination.Write(ctx, recs) } diff --git a/destination_middleware_test.go b/destination_middleware_test.go index af5fa3aa..94fe300f 100644 --- a/destination_middleware_test.go +++ b/destination_middleware_test.go @@ -20,6 +20,8 @@ import ( "testing" "time" + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" "github.com/matryer/is" "go.uber.org/mock/gomock" "golang.org/x/time/rate" @@ -32,7 +34,7 @@ func TestDestinationWithBatch_Parameters(t *testing.T) { d := DestinationWithBatch{}.Wrap(dst) - want := map[string]Parameter{ + want := config.Parameters{ "foo": { Default: "bar", Description: "baz", @@ -114,7 +116,7 @@ func TestDestinationWithRateLimit_Parameters(t *testing.T) { d := DestinationWithRateLimit{}.Wrap(dst) - want := map[string]Parameter{ + want := config.Parameters{ "foo": { Default: "bar", Description: "baz", @@ -228,13 +230,13 @@ func TestDestinationWithRateLimit_Write(t *testing.T) { }) is.NoErr(err) - recs := []Record{{}, {}} + recs := []opencdc.Record{{}, {}} const tolerance = time.Millisecond * 10 expectWriteAfter := func(delay time.Duration) { start := time.Now() - dst.EXPECT().Write(ctx, recs).Do(func(context.Context, []Record) { + dst.EXPECT().Write(ctx, recs).Do(func(context.Context, []opencdc.Record) { dur := time.Since(start) diff := dur - delay if diff < 0 { @@ -278,7 +280,7 @@ func TestDestinationWithRateLimit_Write_CancelledContext(t *testing.T) { is.NoErr(err) cancel() - _, err = d.Write(ctx, []Record{{}}) + _, err = d.Write(ctx, []opencdc.Record{{}}) is.True(errors.Is(err, ctx.Err())) } @@ -288,22 +290,22 @@ func TestDestinationWithRecordFormat_Configure(t *testing.T) { ctx := context.Background() testCases := []struct { - name string - middleware DestinationWithRecordFormat - have map[string]string - wantFormatter RecordFormatter + name string + middleware DestinationWithRecordFormat + have map[string]string + wantSerializer RecordSerializer }{{ - name: "empty config", - middleware: DestinationWithRecordFormat{}, - have: map[string]string{}, - wantFormatter: defaultFormatter, + name: "empty config", + middleware: DestinationWithRecordFormat{}, + have: map[string]string{}, + wantSerializer: defaultSerializer, }, { name: "valid config", middleware: DestinationWithRecordFormat{}, have: map[string]string{ configDestinationRecordFormat: "debezium/json", }, - wantFormatter: GenericRecordFormatter{ + wantSerializer: GenericRecordSerializer{ Converter: DebeziumConverter{ RawDataKey: debeziumDefaultRawDataKey, }, @@ -321,7 +323,7 @@ func TestDestinationWithRecordFormat_Configure(t *testing.T) { err := d.Configure(ctx, tt.have) is.NoErr(err) - is.Equal(d.formatter, tt.wantFormatter) + is.Equal(d.serializer, tt.wantSerializer) }) } } diff --git a/destination_test.go b/destination_test.go index 726a8ad4..6882a68d 100644 --- a/destination_test.go +++ b/destination_test.go @@ -17,13 +17,12 @@ package sdk import ( "context" "errors" - "fmt" "io" "testing" "time" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" - cpluginv1mock "github.com/conduitio/conduit-connector-protocol/cpluginv1/mock" + "github.com/conduitio/conduit-commons/opencdc" + "github.com/conduitio/conduit-connector-protocol/cplugin" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -44,7 +43,7 @@ func TestDestinationPluginAdapter_Start_OpenContext(t *testing.T) { }) ctx, cancel := context.WithCancel(context.Background()) - _, err := dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err := dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) is.NoErr(gotCtx.Err()) // expected context to be open @@ -75,7 +74,7 @@ func TestDestinationPluginAdapter_Start_ClosedContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err := dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.True(err != nil) is.Equal(err, ctx.Err()) is.Equal(gotCtx.Err(), context.Canceled) @@ -99,7 +98,7 @@ func TestDestinationPluginAdapter_Start_Logger(t *testing.T) { ctx := wantLogger.WithContext(context.Background()) - _, err := dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err := dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) } @@ -110,14 +109,14 @@ func TestDestinationPluginAdapter_Run_Write(t *testing.T) { dstPlugin := NewDestinationPlugin(dst).(*destinationPluginAdapter) - want := Record{ - Position: Position("foo"), - Operation: OperationCreate, + want := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, Metadata: map[string]string{"foo": "bar"}, - Key: RawData("bar"), - Payload: Change{ + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, // create has no before - After: StructuredData{ + After: opencdc.StructuredData{ "x": "y", "z": 3, }, @@ -127,14 +126,14 @@ func TestDestinationPluginAdapter_Run_Write(t *testing.T) { dst.EXPECT().Parameters() dst.EXPECT().Configure(gomock.Any(), map[string]string{}).Return(nil) dst.EXPECT().Open(gomock.Any()).Return(nil) - dst.EXPECT().Write(gomock.Any(), []Record{want}).Return(1, nil).Times(10) - - stream, reqStream, respStream := newDestinationRunStreamMock(ctrl) + dst.EXPECT().Write(gomock.Any(), []opencdc.Record{want}).Return(1, nil).Times(10) ctx := context.Background() - _, err := dstPlugin.Configure(ctx, cpluginv1.DestinationConfigureRequest{Config: map[string]string{}}) + stream := NewInMemoryDestinationRunStream(ctx) + + _, err := dstPlugin.Configure(ctx, cplugin.DestinationConfigureRequest{Config: map[string]string{}}) is.NoErr(err) - _, err = dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err = dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) runDone := make(chan struct{}) @@ -145,29 +144,22 @@ func TestDestinationPluginAdapter_Run_Write(t *testing.T) { }() // write 10 records + clientStream := stream.Client() for i := 0; i < 10; i++ { - reqStream <- cpluginv1.DestinationRunRequest{ - Record: cpluginv1.Record{ - Position: want.Position, - Operation: cpluginv1.Operation(want.Operation), - Metadata: want.Metadata, - Key: cpluginv1.RawData(want.Key.(RawData)), - Payload: cpluginv1.Change{ - Before: nil, // create has no before - After: cpluginv1.StructuredData(want.Payload.After.(StructuredData)), - }, - }, - } - resp := <-respStream - is.Equal(resp, cpluginv1.DestinationRunResponse{ - AckPosition: want.Position, - Error: "", + err = clientStream.Send(cplugin.DestinationRunRequest{Records: []opencdc.Record{want}}) + is.NoErr(err) + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.DestinationRunResponse{ + Acks: []cplugin.DestinationRunResponseAck{{ + Position: want.Position, + Error: "", + }}, }) } // close stream - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit <-runDone @@ -182,14 +174,14 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Success(t *testing.T) { DestinationWithMiddleware(dst, DestinationWithBatch{}), ).(*destinationPluginAdapter) - want := Record{ - Position: Position("foo"), - Operation: OperationCreate, + want := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, Metadata: map[string]string{"foo": "bar"}, - Key: RawData("bar"), - Payload: Change{ + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, // create has no before - After: StructuredData{ + After: opencdc.StructuredData{ "x": "y", "z": 3, }, @@ -204,14 +196,14 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Success(t *testing.T) { dst.EXPECT().Parameters() dst.EXPECT().Configure(gomock.Any(), batchConfig).Return(nil) dst.EXPECT().Open(gomock.Any()).Return(nil) - dst.EXPECT().Write(gomock.Any(), []Record{want, want, want, want, want}).Return(5, nil) - - stream, reqStream, respStream := newDestinationRunStreamMock(ctrl) + dst.EXPECT().Write(gomock.Any(), []opencdc.Record{want, want, want, want, want}).Return(5, nil) ctx := context.Background() - _, err := dstPlugin.Configure(ctx, cpluginv1.DestinationConfigureRequest{Config: batchConfig}) + stream := NewInMemoryDestinationRunStream(ctx) + + _, err := dstPlugin.Configure(ctx, cplugin.DestinationConfigureRequest{Config: batchConfig}) is.NoErr(err) - _, err = dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err = dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) runDone := make(chan struct{}) @@ -222,31 +214,24 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Success(t *testing.T) { }() // write 5 records + clientStream := stream.Client() for i := 0; i < 5; i++ { - reqStream <- cpluginv1.DestinationRunRequest{ - Record: cpluginv1.Record{ - Position: want.Position, - Operation: cpluginv1.Operation(want.Operation), - Metadata: want.Metadata, - Key: cpluginv1.RawData(want.Key.(RawData)), - Payload: cpluginv1.Change{ - Before: nil, // create has no before - After: cpluginv1.StructuredData(want.Payload.After.(StructuredData)), - }, - }, - } + err = clientStream.Send(cplugin.DestinationRunRequest{Records: []opencdc.Record{want}}) + is.NoErr(err) } for i := 0; i < 5; i++ { - resp := <-respStream - is.Equal(resp, cpluginv1.DestinationRunResponse{ - AckPosition: want.Position, - Error: "", + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.DestinationRunResponse{ + Acks: []cplugin.DestinationRunResponseAck{{ + Position: want.Position, + Error: "", + }}, }) } // close stream - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit <-runDone @@ -261,14 +246,14 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Partial(t *testing.T) { DestinationWithMiddleware(dst, DestinationWithBatch{}), ).(*destinationPluginAdapter) - want := Record{ - Position: Position("foo"), - Operation: OperationCreate, + want := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, Metadata: map[string]string{"foo": "bar"}, - Key: RawData("bar"), - Payload: Change{ + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, // create has no before - After: StructuredData{ + After: opencdc.StructuredData{ "x": "y", "z": 3, }, @@ -284,14 +269,14 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Partial(t *testing.T) { dst.EXPECT().Parameters() dst.EXPECT().Configure(gomock.Any(), batchConfig).Return(nil) dst.EXPECT().Open(gomock.Any()).Return(nil) - dst.EXPECT().Write(gomock.Any(), []Record{want, want, want, want, want}).Return(3, wantErr) // only 3 records are written - - stream, reqStream, respStream := newDestinationRunStreamMock(ctrl) + dst.EXPECT().Write(gomock.Any(), []opencdc.Record{want, want, want, want, want}).Return(3, wantErr) // only 3 records are written ctx := context.Background() - _, err := dstPlugin.Configure(ctx, cpluginv1.DestinationConfigureRequest{Config: batchConfig}) + stream := NewInMemoryDestinationRunStream(ctx) + + _, err := dstPlugin.Configure(ctx, cplugin.DestinationConfigureRequest{Config: batchConfig}) is.NoErr(err) - _, err = dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err = dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) runDone := make(chan struct{}) @@ -302,38 +287,34 @@ func TestDestinationPluginAdapter_Run_WriteBatch_Partial(t *testing.T) { }() // write 5 records + clientStream := stream.Client() for i := 0; i < 5; i++ { - reqStream <- cpluginv1.DestinationRunRequest{ - Record: cpluginv1.Record{ - Position: want.Position, - Operation: cpluginv1.Operation(want.Operation), - Metadata: want.Metadata, - Key: cpluginv1.RawData(want.Key.(RawData)), - Payload: cpluginv1.Change{ - Before: nil, // create has no before - After: cpluginv1.StructuredData(want.Payload.After.(StructuredData)), - }, - }, - } + err = clientStream.Send(cplugin.DestinationRunRequest{Records: []opencdc.Record{want}}) + is.NoErr(err) } for i := 0; i < 3; i++ { - resp := <-respStream - is.Equal(resp, cpluginv1.DestinationRunResponse{ - AckPosition: want.Position, - Error: "", + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.DestinationRunResponse{ + Acks: []cplugin.DestinationRunResponseAck{{ + Position: want.Position, + Error: "", + }}, }) } for i := 0; i < 2; i++ { - resp := <-respStream - is.Equal(resp, cpluginv1.DestinationRunResponse{ - AckPosition: want.Position, - Error: wantErr.Error(), + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.DestinationRunResponse{ + Acks: []cplugin.DestinationRunResponseAck{{ + Position: want.Position, + Error: wantErr.Error(), + }}, }) } // close stream - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit <-runDone @@ -346,7 +327,7 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { dstPlugin := NewDestinationPlugin(dst).(*destinationPluginAdapter) - lastRecord := Record{Position: Position("foo")} + lastRecord := opencdc.Record{Position: opencdc.Position("foo")} // ackFunc stores the ackFunc so it can be called at a later time dst.EXPECT().Parameters() @@ -354,12 +335,12 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { dst.EXPECT().Open(gomock.Any()).Return(nil) dst.EXPECT().Write(gomock.Any(), gomock.Any()).Return(1, nil) - stream, reqStream, respStream := newDestinationRunStreamMock(ctrl) - ctx := context.Background() - _, err := dstPlugin.Configure(ctx, cpluginv1.DestinationConfigureRequest{Config: map[string]string{}}) + stream := NewInMemoryDestinationRunStream(ctx) + + _, err := dstPlugin.Configure(ctx, cplugin.DestinationConfigureRequest{Config: map[string]string{}}) is.NoErr(err) - _, err = dstPlugin.Start(ctx, cpluginv1.DestinationStartRequest{}) + _, err = dstPlugin.Start(ctx, cplugin.DestinationStartRequest{}) is.NoErr(err) runDone := make(chan struct{}) @@ -377,7 +358,7 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { defer close(stopDone) _, err := dstPlugin.Stop( context.Background(), - cpluginv1.DestinationStopRequest{LastPosition: lastRecord.Position}, + cplugin.DestinationStopRequest{LastPosition: lastRecord.Position}, ) is.NoErr(err) }() @@ -390,9 +371,9 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { } // send last record - reqStream <- cpluginv1.DestinationRunRequest{ - Record: cpluginv1.Record{Position: lastRecord.Position}, - } + clientStream := stream.Client() + err = clientStream.Send(cplugin.DestinationRunRequest{Records: []opencdc.Record{lastRecord}}) + is.NoErr(err) // stop should still block since acknowledgment wasn't sent back yet select { @@ -403,7 +384,8 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { } // let's receive the ack now - <-respStream + _, err = clientStream.Recv() + is.NoErr(err) select { case <-stopDone: @@ -413,8 +395,7 @@ func TestDestinationPluginAdapter_Stop_AwaitLastRecord(t *testing.T) { } // close stream at the end - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit <-runDone @@ -431,7 +412,7 @@ func TestDestinationPluginAdapter_LifecycleOnCreated(t *testing.T) { want := map[string]string{"foo": "bar"} dst.EXPECT().LifecycleOnCreated(ctx, want).Return(nil) - req := cpluginv1.DestinationLifecycleOnCreatedRequest{Config: want} + req := cplugin.DestinationLifecycleOnCreatedRequest{Config: want} _, err := dstPlugin.LifecycleOnCreated(ctx, req) is.NoErr(err) } @@ -448,7 +429,7 @@ func TestDestinationPluginAdapter_LifecycleOnUpdated(t *testing.T) { wantAfter := map[string]string{"foo": "baz"} dst.EXPECT().LifecycleOnUpdated(ctx, wantBefore, wantAfter).Return(nil) - req := cpluginv1.DestinationLifecycleOnUpdatedRequest{ + req := cplugin.DestinationLifecycleOnUpdatedRequest{ ConfigBefore: wantBefore, ConfigAfter: wantAfter, } @@ -467,46 +448,7 @@ func TestDestinationPluginAdapter_LifecycleOnDeleted(t *testing.T) { want := map[string]string{"foo": "bar"} dst.EXPECT().LifecycleOnDeleted(ctx, want).Return(nil) - req := cpluginv1.DestinationLifecycleOnDeletedRequest{Config: want} + req := cplugin.DestinationLifecycleOnDeletedRequest{Config: want} _, err := dstPlugin.LifecycleOnDeleted(ctx, req) is.NoErr(err) } - -func newDestinationRunStreamMock( - ctrl *gomock.Controller, -) ( - *cpluginv1mock.DestinationRunStream, - chan cpluginv1.DestinationRunRequest, - chan cpluginv1.DestinationRunResponse, -) { - stream := cpluginv1mock.NewDestinationRunStream(ctrl) - - reqStream := make(chan cpluginv1.DestinationRunRequest) - respStream := make(chan cpluginv1.DestinationRunResponse) - - stream.EXPECT().Send(gomock.Any()). - DoAndReturn(func(resp cpluginv1.DestinationRunResponse) (err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - err, ok = r.(error) - if !ok { - err = fmt.Errorf("%+v", r) - } - } - }() - respStream <- resp - return nil - }).AnyTimes() - - stream.EXPECT().Recv(). - DoAndReturn(func() (cpluginv1.DestinationRunRequest, error) { - req, ok := <-reqStream - if !ok { - return cpluginv1.DestinationRunRequest{}, io.EOF - } - return req, nil - }).AnyTimes() - - return stream, reqStream, respStream -} diff --git a/error.go b/error.go index 0578ce63..596b50cf 100644 --- a/error.go +++ b/error.go @@ -24,8 +24,4 @@ var ( // ErrUnimplemented is returned in functions of plugins that don't implement // a certain method. ErrUnimplemented = errors.New("the connector plugin does not implement this action, please check the source code of the connector and make sure all required connector methods are implemented") - - // ErrMetadataFieldNotFound is returned in metadata utility functions when a - // metadata field is not found. - ErrMetadataFieldNotFound = errors.New("metadata field not found") ) diff --git a/go.mod b/go.mod index 93b63808..065979d8 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.0 require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/conduitio/conduit-commons v0.2.0 - github.com/conduitio/conduit-connector-protocol v0.6.0 + github.com/conduitio/conduit-connector-protocol v0.6.1-0.20240522160346-94685acd7300 github.com/goccy/go-json v0.10.3 github.com/google/uuid v1.6.0 github.com/jpillora/backoff v1.0.0 @@ -15,23 +15,22 @@ require ( github.com/rs/zerolog v1.33.0 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.4.0 - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f golang.org/x/time v0.5.0 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 ) require ( github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect - github.com/fatih/color v1.16.0 // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/frankban/quicktest v1.14.6 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect - github.com/hashicorp/go-plugin v1.6.0 // indirect + github.com/hashicorp/go-plugin v1.6.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect - github.com/jhump/protoreflect v1.15.6 // indirect + github.com/jhump/protoreflect v1.16.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -42,11 +41,11 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.5.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index c4d75880..f9881cd3 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,24 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= -github.com/bufbuild/protocompile v0.9.0 h1:DI8qLG5PEO0Mu1Oj51YFPqtx6I3qYXUAhJVJ/IzAVl0= -github.com/bufbuild/protocompile v0.9.0/go.mod h1:s89m1O8CqSYpyE/YaSGtg1r1YFMF5nLTwh4vlj6O444= +github.com/bufbuild/protocompile v0.13.1-0.20240510201809-752249dfc37f h1:cQLFPZXf32tbTLzUTg0n69Vi5kddhUiZMzpMzKRW0XU= +github.com/bufbuild/protocompile v0.13.1-0.20240510201809-752249dfc37f/go.mod h1:QJcgsTVPSBEMt+/3i2M/RpwjZc+DAXyPPDg0slmMk4c= github.com/conduitio/conduit-commons v0.2.0 h1:TMpVGXi0Wski537qLAyQWdGjuGHEhaZxOS5L90pZJSQ= github.com/conduitio/conduit-commons v0.2.0/go.mod h1:i7Q2jm7FBSi2zj1/4MCsFD1hIKAbvamlNtSQfkhUTiY= -github.com/conduitio/conduit-connector-protocol v0.6.0 h1:2gMOCOpa+c97CHIpZv7Niu3V4o5UgRr6fzj9kzfRV7o= -github.com/conduitio/conduit-connector-protocol v0.6.0/go.mod h1:3mo59xYX9etFoR3n82R7J50La1iWK+Vm63H8z2wo4QM= +github.com/conduitio/conduit-connector-protocol v0.6.1-0.20240522160346-94685acd7300 h1:fQYJ+e37UGgUyGA7mofJtnLs79O8qKXWjBAt9Gr9NlE= +github.com/conduitio/conduit-connector-protocol v0.6.1-0.20240522160346-94685acd7300/go.mod h1:SKioiokwG3wfHt/Aj1rlVjQoBgZQwmwNl2pvqaVhjws= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= @@ -33,16 +34,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= -github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= +github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI= +github.com/hashicorp/go-plugin v1.6.1/go.mod h1:XPHFku2tFo3o3QKFgSYo+cghcUhw1NA1hZyMK0PWAw0= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/jhump/protoreflect v1.15.6 h1:WMYJbw2Wo+KOWwZFvgY0jMoVHM6i4XIvRs2RcBj5VmI= -github.com/jhump/protoreflect v1.15.6/go.mod h1:jCHoyYQIJnaabEYnbGwyo9hUqfyUMTbJw/tAut5t97E= +github.com/jhump/protoreflect v1.16.0 h1:54fZg+49widqXYQ0b+usAFHbMkBGR4PpXrsHc8+TBDg= +github.com/jhump/protoreflect v1.16.0/go.mod h1:oYPd7nPvcBw/5wlDfm/AVmU9zH9BgqGCI469pGxfj/8= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -100,17 +101,15 @@ go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -127,8 +126,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -136,20 +135,20 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= diff --git a/kafkaconnect/schema_test.go b/kafkaconnect/schema_test.go index 9a029cf1..0a4b474d 100644 --- a/kafkaconnect/schema_test.go +++ b/kafkaconnect/schema_test.go @@ -15,9 +15,8 @@ package kafkaconnect import ( - "testing" - "encoding/json" + "testing" "github.com/matryer/is" ) diff --git a/metadata.go b/metadata.go deleted file mode 100644 index 41120dc3..00000000 --- a/metadata.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "fmt" - "strconv" - "time" - - "github.com/conduitio/conduit-connector-protocol/cpluginv1" -) - -const ( - // MetadataOpenCDCVersion is a Record.Metadata key for the version of the - // OpenCDC format (e.g. "v1"). This field exists to ensure the OpenCDC - // format version can be easily identified in case the record gets marshaled - // into a different untyped format (e.g. JSON). - MetadataOpenCDCVersion = cpluginv1.MetadataOpenCDCVersion - - // MetadataCreatedAt is a Record.Metadata key for the time when the record - // was created in the 3rd party system. The expected format is a unix - // timestamp in nanoseconds. - MetadataCreatedAt = cpluginv1.MetadataCreatedAt - // MetadataReadAt is a Record.Metadata key for the time when the record was - // read from the 3rd party system. The expected format is a unix timestamp - // in nanoseconds. - MetadataReadAt = cpluginv1.MetadataReadAt - // MetadataCollection is a Record.Metadata key for the name of the collection - // where the record originated from and/or where it should be stored. - MetadataCollection = cpluginv1.MetadataCollection - - // MetadataConduitSourcePluginName is a Record.Metadata key for the name of - // the source plugin that created this record. - MetadataConduitSourcePluginName = cpluginv1.MetadataConduitSourcePluginName - // MetadataConduitSourcePluginVersion is a Record.Metadata key for the - // version of the source plugin that created this record. - MetadataConduitSourcePluginVersion = cpluginv1.MetadataConduitSourcePluginVersion - // MetadataConduitDestinationPluginName is a Record.Metadata key for the - // name of the destination plugin that has written this record - // (only available in records once they are written by a destination). - MetadataConduitDestinationPluginName = cpluginv1.MetadataConduitDestinationPluginName - // MetadataConduitDestinationPluginVersion is a Record.Metadata key for the - // version of the destination plugin that has written this record - // (only available in records once they are written by a destination). - MetadataConduitDestinationPluginVersion = cpluginv1.MetadataConduitDestinationPluginVersion - - // MetadataConduitSourceConnectorID is a Record.Metadata key for the ID of - // the source connector that produced this record. - MetadataConduitSourceConnectorID = cpluginv1.MetadataConduitSourceConnectorID - // MetadataConduitDLQNackError is a Record.Metadata key for the error that - // caused a record to be nacked and pushed to the dead-letter queue. - MetadataConduitDLQNackError = cpluginv1.MetadataConduitDLQNackError - // MetadataConduitDLQNackNodeID is a Record.Metadata key for the ID of the - // internal node that nacked the record. - MetadataConduitDLQNackNodeID = cpluginv1.MetadataConduitDLQNackNodeID -) - -// SetOpenCDCVersion sets the metadata value for key MetadataVersion to the -// current version of OpenCDC used. -func (m Metadata) SetOpenCDCVersion() { - m[MetadataOpenCDCVersion] = cpluginv1.OpenCDCVersion -} - -// GetOpenCDCVersion returns the value for key -// MetadataOpenCDCVersion. If the value is does not exist or is empty the -// function returns ErrMetadataFieldNotFound. -func (m Metadata) GetOpenCDCVersion() (string, error) { - return m.getValue(MetadataOpenCDCVersion) -} - -// GetCreatedAt parses the value for key MetadataCreatedAt as a unix -// timestamp. If the value does not exist or the value is empty the function -// returns ErrMetadataFieldNotFound. If the value is not a valid unix timestamp -// in nanoseconds the function returns an error. -func (m Metadata) GetCreatedAt() (time.Time, error) { - raw, err := m.getValue(MetadataCreatedAt) - if err != nil { - return time.Time{}, err - } - - unixNano, err := strconv.ParseInt(raw, 10, 64) - if err != nil { - return time.Time{}, fmt.Errorf("failed to parse value for %q: %w", MetadataCreatedAt, err) - } - - return time.Unix(0, unixNano), nil -} - -// SetCreatedAt sets the metadata value for key MetadataCreatedAt as a -// unix timestamp in nanoseconds. -func (m Metadata) SetCreatedAt(createdAt time.Time) { - m[MetadataCreatedAt] = strconv.FormatInt(createdAt.UnixNano(), 10) -} - -// GetReadAt parses the value for key MetadataReadAt as a unix -// timestamp. If the value does not exist or the value is empty the function -// returns ErrMetadataFieldNotFound. If the value is not a valid unix timestamp -// in nanoseconds the function returns an error. -func (m Metadata) GetReadAt() (time.Time, error) { - raw, err := m.getValue(MetadataReadAt) - if err != nil { - return time.Time{}, err - } - - unixNano, err := strconv.ParseInt(raw, 10, 64) - if err != nil { - return time.Time{}, fmt.Errorf("failed to parse value for %q: %w", MetadataReadAt, err) - } - - return time.Unix(0, unixNano), nil -} - -// SetReadAt sets the metadata value for key MetadataReadAt as a unix -// timestamp in nanoseconds. -func (m Metadata) SetReadAt(createdAt time.Time) { - m[MetadataReadAt] = strconv.FormatInt(createdAt.UnixNano(), 10) -} - -// GetCollection returns the value for key MetadataCollection. If the value does -// not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetCollection() (string, error) { - return m.getValue(MetadataCollection) -} - -// SetCollection sets the metadata value for key MetadataCollection. -func (m Metadata) SetCollection(collection string) { - m[MetadataCollection] = collection -} - -// GetConduitSourcePluginName returns the value for key -// MetadataConduitSourcePluginName. If the value does not exist or is empty the -// function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitSourcePluginName() (string, error) { - return m.getValue(MetadataConduitSourcePluginName) -} - -// SetConduitSourcePluginName sets the metadata value for key -// MetadataConduitSourcePluginName. -func (m Metadata) SetConduitSourcePluginName(name string) { - m[MetadataConduitSourcePluginName] = name -} - -// GetConduitSourcePluginVersion returns the value for key -// MetadataConduitSourcePluginVersion. If the value does not exist or is empty -// the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitSourcePluginVersion() (string, error) { - return m.getValue(MetadataConduitSourcePluginVersion) -} - -// SetConduitSourcePluginVersion sets the metadata value for key -// MetadataConduitSourcePluginVersion. -func (m Metadata) SetConduitSourcePluginVersion(version string) { - m[MetadataConduitSourcePluginVersion] = version -} - -// GetConduitDestinationPluginName returns the value for key -// MetadataConduitDestinationPluginName. If the value does not exist or is empty -// the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitDestinationPluginName() (string, error) { - return m.getValue(MetadataConduitDestinationPluginName) -} - -// SetConduitDestinationPluginName sets the metadata value for key -// MetadataConduitDestinationPluginName. -func (m Metadata) SetConduitDestinationPluginName(name string) { - m[MetadataConduitDestinationPluginName] = name -} - -// GetConduitDestinationPluginVersion returns the value for key -// MetadataConduitDestinationPluginVersion. If the value does not exist or is -// empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitDestinationPluginVersion() (string, error) { - return m.getValue(MetadataConduitDestinationPluginVersion) -} - -// SetConduitDestinationPluginVersion sets the metadata value for key -// MetadataConduitDestinationPluginVersion. -func (m Metadata) SetConduitDestinationPluginVersion(version string) { - m[MetadataConduitDestinationPluginVersion] = version -} - -// GetConduitSourceConnectorID returns the value for key -// MetadataConduitSourceConnectorID. If the value does not exist or is empty the -// function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitSourceConnectorID() (string, error) { - return m.getValue(MetadataConduitSourceConnectorID) -} - -// SetConduitSourceConnectorID sets the metadata value for key -// MetadataConduitSourceConnectorID. -func (m Metadata) SetConduitSourceConnectorID(id string) { - m[MetadataConduitSourceConnectorID] = id -} - -// GetConduitDLQNackError returns the value for key -// MetadataConduitDLQNackError. If the value does not exist or is empty the -// function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitDLQNackError() (string, error) { - return m.getValue(MetadataConduitDLQNackError) -} - -// SetConduitDLQNackError sets the metadata value for key -// MetadataConduitDLQNackError. -func (m Metadata) SetConduitDLQNackError(err string) { - m[MetadataConduitDLQNackError] = err -} - -// GetConduitDLQNackNodeID returns the value for key -// MetadataConduitDLQNackNodeID. If the value does not exist or is empty the -// function returns ErrMetadataFieldNotFound. -func (m Metadata) GetConduitDLQNackNodeID() (string, error) { - return m.getValue(MetadataConduitDLQNackNodeID) -} - -// SetConduitDLQNackNodeID sets the metadata value for key -// MetadataConduitDLQNackNodeID. -func (m Metadata) SetConduitDLQNackNodeID(id string) { - m[MetadataConduitDLQNackNodeID] = id -} - -// getValue returns the value for a specific key. If the value is does -// not exist or is empty the function returns ErrMetadataFieldNotFound. -func (m Metadata) getValue(key string) (string, error) { - str := m[key] - if str == "" { - return "", fmt.Errorf("failed to get value for %q: %w", key, ErrMetadataFieldNotFound) - } - return str, nil -} diff --git a/mock_destination_test.go b/mock_destination_test.go index 5270e80d..1fb163c7 100644 --- a/mock_destination_test.go +++ b/mock_destination_test.go @@ -10,6 +10,8 @@ import ( context "context" reflect "reflect" + config "github.com/conduitio/conduit-commons/config" + opencdc "github.com/conduitio/conduit-commons/opencdc" gomock "go.uber.org/mock/gomock" ) @@ -107,10 +109,10 @@ func (mr *MockDestinationMockRecorder) Open(arg0 any) *gomock.Call { } // Parameters mocks base method. -func (m *MockDestination) Parameters() map[string]Parameter { +func (m *MockDestination) Parameters() config.Parameters { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Parameters") - ret0, _ := ret[0].(map[string]Parameter) + ret0, _ := ret[0].(config.Parameters) return ret0 } @@ -135,7 +137,7 @@ func (mr *MockDestinationMockRecorder) Teardown(arg0 any) *gomock.Call { } // Write mocks base method. -func (m *MockDestination) Write(arg0 context.Context, arg1 []Record) (int, error) { +func (m *MockDestination) Write(arg0 context.Context, arg1 []opencdc.Record) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Write", arg0, arg1) ret0, _ := ret[0].(int) diff --git a/mock_source_test.go b/mock_source_test.go index 31b44921..30e1fa48 100644 --- a/mock_source_test.go +++ b/mock_source_test.go @@ -10,6 +10,8 @@ import ( context "context" reflect "reflect" + config "github.com/conduitio/conduit-commons/config" + opencdc "github.com/conduitio/conduit-commons/opencdc" gomock "go.uber.org/mock/gomock" ) @@ -37,7 +39,7 @@ func (m *MockSource) EXPECT() *MockSourceMockRecorder { } // Ack mocks base method. -func (m *MockSource) Ack(arg0 context.Context, arg1 Position) error { +func (m *MockSource) Ack(arg0 context.Context, arg1 opencdc.Position) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Ack", arg0, arg1) ret0, _ := ret[0].(error) @@ -107,7 +109,7 @@ func (mr *MockSourceMockRecorder) LifecycleOnUpdated(arg0, arg1, arg2 any) *gomo } // Open mocks base method. -func (m *MockSource) Open(arg0 context.Context, arg1 Position) error { +func (m *MockSource) Open(arg0 context.Context, arg1 opencdc.Position) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Open", arg0, arg1) ret0, _ := ret[0].(error) @@ -121,10 +123,10 @@ func (mr *MockSourceMockRecorder) Open(arg0, arg1 any) *gomock.Call { } // Parameters mocks base method. -func (m *MockSource) Parameters() map[string]Parameter { +func (m *MockSource) Parameters() config.Parameters { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Parameters") - ret0, _ := ret[0].(map[string]Parameter) + ret0, _ := ret[0].(config.Parameters) return ret0 } @@ -135,10 +137,10 @@ func (mr *MockSourceMockRecorder) Parameters() *gomock.Call { } // Read mocks base method. -func (m *MockSource) Read(arg0 context.Context) (Record, error) { +func (m *MockSource) Read(arg0 context.Context) (opencdc.Record, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Read", arg0) - ret0, _ := ret[0].(Record) + ret0, _ := ret[0].(opencdc.Record) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/operation_string.go b/operation_string.go deleted file mode 100644 index 3cffdbbd..00000000 --- a/operation_string.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated by "stringer -type=Operation -linecomment"; DO NOT EDIT. - -package sdk - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[OperationCreate-1] - _ = x[OperationUpdate-2] - _ = x[OperationDelete-3] - _ = x[OperationSnapshot-4] -} - -const _Operation_name = "createupdatedeletesnapshot" - -var _Operation_index = [...]uint8{0, 6, 12, 18, 26} - -func (i Operation) String() string { - i -= 1 - if i < 0 || i >= Operation(len(_Operation_index)-1) { - return "Operation(" + strconv.FormatInt(int64(i+1), 10) + ")" - } - return _Operation_name[_Operation_index[i]:_Operation_index[i+1]] -} diff --git a/record.go b/record.go deleted file mode 100644 index 2af374e9..00000000 --- a/record.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:generate stringer -type=Operation -linecomment - -package sdk - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/conduitio/conduit-connector-protocol/cpluginv1" - "github.com/goccy/go-json" -) - -const ( - OperationCreate Operation = iota + 1 // create - OperationUpdate // update - OperationDelete // delete - OperationSnapshot // snapshot -) - -// Operation defines what triggered the creation of a record. -type Operation int - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - var cTypes [1]struct{} - _ = cTypes[int(OperationCreate)-int(cpluginv1.OperationCreate)] - _ = cTypes[int(OperationUpdate)-int(cpluginv1.OperationUpdate)] - _ = cTypes[int(OperationDelete)-int(cpluginv1.OperationDelete)] - _ = cTypes[int(OperationSnapshot)-int(cpluginv1.OperationSnapshot)] -} - -func (i Operation) MarshalText() ([]byte, error) { - return []byte(i.String()), nil -} - -func (i *Operation) UnmarshalText(b []byte) error { - if len(b) == 0 { - return nil // empty string, do nothing - } - - switch string(b) { - case OperationCreate.String(): - *i = OperationCreate - case OperationUpdate.String(): - *i = OperationUpdate - case OperationDelete.String(): - *i = OperationDelete - case OperationSnapshot.String(): - *i = OperationSnapshot - default: - // it's not a known operation, but we also allow Operation(int) - valIntRaw := strings.TrimSuffix(strings.TrimPrefix(string(b), "Operation("), ")") - valInt, err := strconv.Atoi(valIntRaw) - if err != nil { - return fmt.Errorf("unknown operation %q", b) - } - *i = Operation(valInt) - } - - return nil -} - -// Record represents a single data record produced by a source and/or consumed -// by a destination connector. -type Record struct { - // Position uniquely represents the record. - Position Position `json:"position"` - // Operation defines what triggered the creation of a record. There are four - // possibilities: create, update, delete or snapshot. The first three - // operations are encountered during normal CDC operation, while "snapshot" - // is meant to represent records during an initial load. Depending on the - // operation, the record will contain either the payload before the change, - // after the change, or both (see field Payload). - Operation Operation `json:"operation"` - // Metadata contains additional information regarding the record. - Metadata Metadata `json:"metadata"` - - // Key represents a value that should identify the entity (e.g. database - // row). - Key Data `json:"key"` - // Payload holds the payload change (data before and after the operation - // occurred). - Payload Change `json:"payload"` - - formatter RecordFormatter -} - -type Metadata map[string]string - -// Bytes returns the JSON encoding of the Record. -func (r Record) Bytes() []byte { - var b []byte - var err error - if r.formatter == nil { - b, err = defaultFormatter.Format(r) - } else { - b, err = r.formatter.Format(r) - } - if err != nil { - err = fmt.Errorf("error while formatting Record: %w", err) - Logger(context.Background()).Err(err).Msg("falling back to JSON format") - b, err = json.Marshal(r) - if err != nil { - // Unlikely to happen, we receive content from a plugin through GRPC. - // If the content could be marshaled as protobuf it can be as JSON. - panic(fmt.Errorf("error while marshaling Record as JSON: %w", err)) - } - } - return b -} - -type Change struct { - // Before contains the data before the operation occurred. This field is - // optional and should only be populated for operations OperationUpdate - // OperationDelete (if the system supports fetching the data before the - // operation). - Before Data `json:"before"` - // After contains the data after the operation occurred. This field should - // be populated for all operations except OperationDelete. - After Data `json:"after"` -} - -// Position is a unique identifier for a record. It is the responsibility of the -// Source to choose and assign record positions, it can freely choose a format -// that makes sense and contains everything needed to restart a pipeline at a -// certain position. -type Position []byte - -// Data is a structure that contains some bytes. The only structs implementing -// Data are RawData and StructuredData. -type Data interface { - isData() - Bytes() []byte -} - -// RawData contains unstructured data in form of a byte slice. -type RawData []byte - -func (RawData) isData() {} - -// Bytes simply casts RawData to a byte slice. -func (d RawData) Bytes() []byte { - return d -} - -// StructuredData contains data in form of a map with string keys and arbitrary -// values. -type StructuredData map[string]interface{} - -func (StructuredData) isData() {} - -// Bytes returns the JSON encoding of the map. -func (d StructuredData) Bytes() []byte { - b, err := json.Marshal(d) - if err != nil { - // Unlikely to happen, we receive content from a plugin through GRPC. - // If the content could be marshaled as protobuf it can be as JSON. - panic(fmt.Errorf("error while marshaling StructuredData as JSON: %w", err)) - } - return b -} diff --git a/record_formatter.go b/record_serializer.go similarity index 74% rename from record_formatter.go rename to record_serializer.go index 886da922..3e8fd752 100644 --- a/record_formatter.go +++ b/record_serializer.go @@ -21,22 +21,23 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" + "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit-connector-sdk/kafkaconnect" "github.com/goccy/go-json" ) -// RecordFormatter is a type that can format a record to bytes. It's used in +// RecordSerializer is a type that can format a record to bytes. It's used in // destination connectors to change the output structure and format. -type RecordFormatter interface { +type RecordSerializer interface { Name() string - Configure(string) (RecordFormatter, error) - Format(Record) ([]byte, error) + Configure(string) (RecordSerializer, error) + opencdc.RecordSerializer } var ( - defaultConverter = OpenCDCConverter{} - defaultEncoder = JSONEncoder{} - defaultFormatter = GenericRecordFormatter{ + defaultConverter = OpenCDCConverter{} + defaultEncoder = JSONEncoder{} + defaultSerializer = GenericRecordSerializer{ Converter: defaultConverter, Encoder: defaultEncoder, } @@ -48,9 +49,9 @@ const ( recordFormatOptionsPairSeparator = "=" // e.g. opt1=val1 ) -// GenericRecordFormatter is a formatter that uses a Converter and Encoder to -// format a record. -type GenericRecordFormatter struct { +// GenericRecordSerializer is a serializer that uses a Converter and Encoder to +// serialize a record. +type GenericRecordSerializer struct { Converter Encoder } @@ -61,7 +62,7 @@ type GenericRecordFormatter struct { type Converter interface { Name() string Configure(map[string]string) (Converter, error) - Convert(Record) (any, error) + Convert(opencdc.Record) (any, error) } // Encoder is a type that can encode a random struct into a byte slice. It's @@ -73,13 +74,13 @@ type Encoder interface { Encode(r any) ([]byte, error) } -// Name returns the name of the record formatter combined from the converter +// Name returns the name of the record serializer combined from the converter // name and encoder name. -func (rf GenericRecordFormatter) Name() string { +func (rf GenericRecordSerializer) Name() string { return rf.Converter.Name() + genericRecordFormatSeparator + rf.Encoder.Name() } -func (rf GenericRecordFormatter) Configure(optRaw string) (RecordFormatter, error) { +func (rf GenericRecordSerializer) Configure(optRaw string) (RecordSerializer, error) { opt := rf.parseFormatOptions(optRaw) var err error @@ -94,7 +95,7 @@ func (rf GenericRecordFormatter) Configure(optRaw string) (RecordFormatter, erro return rf, nil } -func (rf GenericRecordFormatter) parseFormatOptions(options string) map[string]string { +func (rf GenericRecordSerializer) parseFormatOptions(options string) map[string]string { options = strings.TrimSpace(options) if len(options) == 0 { return nil @@ -114,8 +115,8 @@ func (rf GenericRecordFormatter) parseFormatOptions(options string) map[string]s return optMap } -// Format converts and encodes record into a byte array. -func (rf GenericRecordFormatter) Format(r Record) ([]byte, error) { +// Serialize converts and encodes record into a byte array. +func (rf GenericRecordSerializer) Serialize(r opencdc.Record) ([]byte, error) { converted, err := rf.Converter.Convert(r) if err != nil { return nil, fmt.Errorf("converter %s failed: %w", rf.Converter.Name(), err) @@ -135,7 +136,7 @@ type OpenCDCConverter struct{} func (c OpenCDCConverter) Name() string { return "opencdc" } func (c OpenCDCConverter) Configure(map[string]string) (Converter, error) { return c, nil } -func (c OpenCDCConverter) Convert(r Record) (any, error) { +func (c OpenCDCConverter) Convert(r opencdc.Record) (any, error) { return r, nil } @@ -158,7 +159,7 @@ func (c DebeziumConverter) Configure(opt map[string]string) (Converter, error) { } return c, nil } -func (c DebeziumConverter) Convert(r Record) (any, error) { +func (c DebeziumConverter) Convert(r opencdc.Record) (any, error) { before, err := c.getStructuredData(r.Payload.Before) if err != nil { return nil, err @@ -188,13 +189,13 @@ func (c DebeziumConverter) Convert(r Record) (any, error) { return e, nil } -func (c DebeziumConverter) getStructuredData(d Data) (StructuredData, error) { +func (c DebeziumConverter) getStructuredData(d opencdc.Data) (opencdc.StructuredData, error) { switch d := d.(type) { case nil: return nil, nil - case StructuredData: + case opencdc.StructuredData: return d, nil - case RawData: + case opencdc.RawData: if len(d) == 0 { return nil, nil } @@ -202,32 +203,32 @@ func (c DebeziumConverter) getStructuredData(d Data) (StructuredData, error) { if err != nil { // we have actually raw data, fall back to artificial structured // data by hoisting it into a field - sd = StructuredData{c.RawDataKey: d.Bytes()} + sd = opencdc.StructuredData{c.RawDataKey: d.Bytes()} } return sd, nil default: return nil, fmt.Errorf("unknown data type: %T", d) } } -func (c DebeziumConverter) parseRawDataAsJSON(d RawData) (StructuredData, error) { +func (c DebeziumConverter) parseRawDataAsJSON(d opencdc.RawData) (opencdc.StructuredData, error) { // We have raw data, we need structured data. // We can do our best and try to convert it if RawData is carrying raw JSON. - var sd StructuredData + var sd opencdc.StructuredData err := json.Unmarshal(d, &sd) if err != nil { return nil, fmt.Errorf("could not convert RawData to StructuredData: %w", err) } return sd, nil } -func (c DebeziumConverter) getDebeziumOp(o Operation) kafkaconnect.DebeziumOp { +func (c DebeziumConverter) getDebeziumOp(o opencdc.Operation) kafkaconnect.DebeziumOp { switch o { - case OperationCreate: + case opencdc.OperationCreate: return kafkaconnect.DebeziumOpCreate - case OperationUpdate: + case opencdc.OperationUpdate: return kafkaconnect.DebeziumOpUpdate - case OperationDelete: + case opencdc.OperationDelete: return kafkaconnect.DebeziumOpDelete - case OperationSnapshot: + case opencdc.OperationSnapshot: return kafkaconnect.DebeziumOpRead } return "" // invalid operation @@ -242,14 +243,14 @@ func (e JSONEncoder) Encode(v any) ([]byte, error) { return json.Marshal(v) } -// TemplateRecordFormatter is a RecordFormatter that formats a record using a Go -// template. -type TemplateRecordFormatter struct { +// TemplateRecordSerializer is a RecordSerializer that serializes a record using +// a Go template. +type TemplateRecordSerializer struct { template *template.Template } -func (e TemplateRecordFormatter) Name() string { return "template" } -func (e TemplateRecordFormatter) Configure(tmpl string) (RecordFormatter, error) { +func (e TemplateRecordSerializer) Name() string { return "template" } +func (e TemplateRecordSerializer) Configure(tmpl string) (RecordSerializer, error) { t := template.New("") t = t.Funcs(sprig.TxtFuncMap()) // inject sprig functions t, err := t.Parse(tmpl) @@ -260,7 +261,7 @@ func (e TemplateRecordFormatter) Configure(tmpl string) (RecordFormatter, error) e.template = t return e, nil } -func (e TemplateRecordFormatter) Format(r Record) ([]byte, error) { +func (e TemplateRecordSerializer) Serialize(r opencdc.Record) ([]byte, error) { var b bytes.Buffer err := e.template.Execute(&b, r) if err != nil { diff --git a/record_formatter_test.go b/record_serializer_test.go similarity index 70% rename from record_formatter_test.go rename to record_serializer_test.go index a7171460..70c8d5d6 100644 --- a/record_formatter_test.go +++ b/record_serializer_test.go @@ -17,6 +17,7 @@ package sdk import ( "testing" + "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit-connector-sdk/kafkaconnect" "github.com/matryer/is" ) @@ -27,14 +28,14 @@ var ( ) func BenchmarkJSONEncoder(b *testing.B) { - rec := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ + rec := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, - After: StructuredData{ + After: opencdc.StructuredData{ "foo": "bar", "baz": "qux", }, @@ -51,14 +52,14 @@ func TestOpenCDCConverter(t *testing.T) { is := is.New(t) var converter OpenCDCConverter - want := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ + want := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, - After: StructuredData{ + After: opencdc.StructuredData{ "foo": "bar", "baz": "qux", }, @@ -77,16 +78,16 @@ func TestDebeziumConverter_Structured(t *testing.T) { }) is.NoErr(err) - r := Record{ - Position: Position("foo"), - Operation: OperationUpdate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ - Before: StructuredData{ + r := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationUpdate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ + Before: opencdc.StructuredData{ "bar": 123, }, - After: StructuredData{ + After: opencdc.StructuredData{ "foo": "bar", "baz": "qux", }, @@ -147,8 +148,8 @@ func TestDebeziumConverter_Structured(t *testing.T) { }}, }, Payload: kafkaconnect.DebeziumPayload{ - Before: r.Payload.Before.(StructuredData), - After: r.Payload.After.(StructuredData), + Before: r.Payload.Before.(opencdc.StructuredData), + After: r.Payload.After.(opencdc.StructuredData), Source: r.Metadata, Op: kafkaconnect.DebeziumOpUpdate, TimestampMillis: 0, @@ -173,13 +174,13 @@ func TestDebeziumConverter_RawData(t *testing.T) { converter, err := DebeziumConverter{}.Configure(map[string]string{}) is.NoErr(err) - r := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ - Before: RawData("foo"), + r := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ + Before: opencdc.RawData("foo"), After: nil, }, } @@ -236,7 +237,7 @@ func TestDebeziumConverter_RawData(t *testing.T) { }}, }, Payload: kafkaconnect.DebeziumPayload{ - Before: StructuredData{"opencdc.rawData": []byte("foo")}, + Before: opencdc.StructuredData{"opencdc.rawData": []byte("foo")}, After: nil, Source: r.Metadata, Op: kafkaconnect.DebeziumOpCreate, @@ -261,14 +262,14 @@ func TestJSONEncoder(t *testing.T) { is := is.New(t) var encoder JSONEncoder - r := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ + r := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, - After: StructuredData{ + After: opencdc.StructuredData{ "foo": "bar", "baz": "qux", }, @@ -281,15 +282,15 @@ func TestJSONEncoder(t *testing.T) { is.Equal(string(got), want) } -func TestTemplateRecordFormatter(t *testing.T) { - r := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ +func TestTemplateRecordSerializer(t *testing.T) { + r := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, + Metadata: opencdc.Metadata{opencdc.MetadataConduitSourcePluginName: "example"}, + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, - After: StructuredData{ + After: opencdc.StructuredData{ "foo": "bar", "baz": "qux", }, @@ -297,7 +298,7 @@ func TestTemplateRecordFormatter(t *testing.T) { } testCases := map[string]struct { - have Record + have opencdc.Record template string want string }{ @@ -305,7 +306,7 @@ func TestTemplateRecordFormatter(t *testing.T) { // output prints the Go record (not very useful, this test case is here to explain the behavior) have: r, template: `{{ . }}`, - want: `{[102 111 111] create map[conduit.source.plugin.name:example] [98 97 114] { map[baz:qux foo:bar]} }`, + want: `{foo create map[conduit.source.plugin.name:example] [98 97 114] { map[baz:qux foo:bar]} }`, }, "json record": { // output should be the same as in format opencdc/json @@ -315,25 +316,25 @@ func TestTemplateRecordFormatter(t *testing.T) { }, "json structured payload": { have: r, - template: `{{ if typeIs "sdk.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, + template: `{{ if typeIs "opencdc.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, want: `{"baz":"qux","foo":"bar"}`, }, "json raw payload": { - have: Record{ - Payload: Change{ - After: RawData("my raw data"), + have: opencdc.Record{ + Payload: opencdc.Change{ + After: opencdc.RawData("my raw data"), }, }, - template: `{{ if typeIs "sdk.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, + template: `{{ if typeIs "opencdc.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, want: `my raw data`, }, "json nil payload": { - have: Record{ - Payload: Change{ + have: opencdc.Record{ + Payload: opencdc.Change{ After: nil, }, }, - template: `{{ if typeIs "sdk.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, + template: `{{ if typeIs "opencdc.RawData" .Payload.After }}{{ printf "%s" .Payload.After }}{{ else }}{{ toJson .Payload.After }}{{ end }}`, want: `null`, }, "map metadata": { @@ -346,12 +347,12 @@ func TestTemplateRecordFormatter(t *testing.T) { for testName, tc := range testCases { t.Run(testName, func(t *testing.T) { is := is.New(t) - var formatter RecordFormatter = TemplateRecordFormatter{} + var serializer RecordSerializer = TemplateRecordSerializer{} - formatter, err := formatter.Configure(tc.template) + serializer, err := serializer.Configure(tc.template) is.NoErr(err) - got, err := formatter.Format(tc.have) + got, err := serializer.Serialize(tc.have) is.NoErr(err) is.Equal(string(got), tc.want) }) diff --git a/record_test.go b/record_test.go deleted file mode 100644 index a09210f7..00000000 --- a/record_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "testing" - - "github.com/matryer/is" -) - -func TestRecord_Bytes_Default(t *testing.T) { - is := is.New(t) - - r := Record{ - Position: Position("foo"), - Operation: OperationCreate, - Metadata: Metadata{MetadataConduitSourcePluginName: "example"}, - Key: RawData("bar"), - Payload: Change{ - Before: nil, - After: StructuredData{ - "foo": "bar", - "baz": "qux", - }, - }, - } - - want := `{"position":"Zm9v","operation":"create","metadata":{"conduit.source.plugin.name":"example"},"key":"YmFy","payload":{"before":null,"after":{"baz":"qux","foo":"bar"}}}` - - got := string(r.Bytes()) - is.Equal(got, want) -} diff --git a/serve.go b/serve.go index 0aa65aca..5b2a694e 100644 --- a/serve.go +++ b/serve.go @@ -19,8 +19,8 @@ import ( "fmt" "os" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" - "github.com/conduitio/conduit-connector-protocol/cpluginv1/server" + "github.com/conduitio/conduit-connector-protocol/cplugin" + "github.com/conduitio/conduit-connector-protocol/cplugin/server" ) // Serve starts the plugin and takes care of its whole lifecycle by blocking @@ -58,10 +58,10 @@ func serve(c Connector) error { } return server.Serve( - func() cpluginv1.SpecifierPlugin { + func() cplugin.SpecifierPlugin { return NewSpecifierPlugin(c.NewSpecification(), c.NewSource(), c.NewDestination()) }, - func() cpluginv1.SourcePlugin { return NewSourcePlugin(c.NewSource()) }, - func() cpluginv1.DestinationPlugin { return NewDestinationPlugin(c.NewDestination()) }, + func() cplugin.SourcePlugin { return NewSourcePlugin(c.NewSource()) }, + func() cplugin.DestinationPlugin { return NewDestinationPlugin(c.NewDestination()) }, ) } diff --git a/source.go b/source.go index 74ff10ff..613e5446 100644 --- a/source.go +++ b/source.go @@ -24,7 +24,8 @@ import ( "time" "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" + "github.com/conduitio/conduit-commons/opencdc" + cplugin "github.com/conduitio/conduit-connector-protocol/cplugin" "github.com/conduitio/conduit-connector-sdk/internal" "github.com/conduitio/conduit-connector-sdk/internal/cchan" "github.com/conduitio/conduit-connector-sdk/internal/csync" @@ -43,7 +44,7 @@ var ( type Source interface { // Parameters is a map of named Parameters that describe how to configure // the Source. - Parameters() map[string]Parameter + Parameters() config.Parameters // Configure is the first function to be called in a connector. It provides the // connector with the configuration that needs to be validated and stored. @@ -63,7 +64,7 @@ type Source interface { // last record that was successfully processed, Source should therefore // start producing records after this position. The context passed to Open // will be cancelled once the plugin receives a stop signal from Conduit. - Open(context.Context, Position) error + Open(context.Context, opencdc.Position) error // Read returns a new Record and is supposed to block until there is either // a new record or the context gets cancelled. It can also return the error @@ -79,14 +80,14 @@ type Source interface { // After Read returns an error the function won't be called again (except if // the error is ErrBackoffRetry, as mentioned above). // Read can be called concurrently with Ack. - Read(context.Context) (Record, error) + Read(context.Context) (opencdc.Record, error) // Ack signals to the implementation that the record with the supplied // position was successfully processed. This method might be called after // the context of Read is already cancelled, since there might be // outstanding acks that need to be delivered. When Teardown is called it is // guaranteed there won't be any more calls to Ack. // Ack can be called concurrently with Read. - Ack(context.Context, Position) error + Ack(context.Context, opencdc.Position) error // Teardown signals to the plugin that there will be no more calls to any // other function. After Teardown returns, the plugin should be ready for a @@ -118,9 +119,9 @@ type Source interface { } // NewSourcePlugin takes a Source and wraps it into an adapter that converts it -// into a cpluginv1.SourcePlugin. If the parameter is nil it will wrap +// into a cplugin.SourcePlugin. If the parameter is nil it will wrap // UnimplementedSource instead. -func NewSourcePlugin(impl Source) cpluginv1.SourcePlugin { +func NewSourcePlugin(impl Source) cplugin.SourcePlugin { if impl == nil { // prevent nil pointers impl = UnimplementedSource{} @@ -137,22 +138,23 @@ type sourcePluginAdapter struct { readDone chan struct{} // lastPosition stores the position of the last record sent to Conduit. - lastPosition Position + lastPosition opencdc.Position openCancel context.CancelFunc readCancel context.CancelFunc t *tomb.Tomb } -func (a *sourcePluginAdapter) Configure(ctx context.Context, req cpluginv1.SourceConfigureRequest) (cpluginv1.SourceConfigureResponse, error) { +func (a *sourcePluginAdapter) Configure(ctx context.Context, req cplugin.SourceConfigureRequest) (cplugin.SourceConfigureResponse, error) { err := a.state.DoWithLock(ctx, internal.DoWithLockOptions{ ExpectedStates: []internal.ConnectorState{internal.StateInitial}, StateBefore: internal.StateConfiguring, StateAfter: internal.StateConfigured, WaitForExpectedState: false, }, func(_ internal.ConnectorState) error { - params := parameters(a.impl.Parameters()).toConfigParameters() + params := a.impl.Parameters() + // TODO should we stop doing this here? The Processor SDK does NOT do this. // sanitize config and apply default values cfg := config.Config(req.Config). Sanitize(). @@ -166,10 +168,10 @@ func (a *sourcePluginAdapter) Configure(ctx context.Context, req cpluginv1.Sourc return errors.Join(err1, err2) }) - return cpluginv1.SourceConfigureResponse{}, err + return cplugin.SourceConfigureResponse{}, err } -func (a *sourcePluginAdapter) Start(ctx context.Context, req cpluginv1.SourceStartRequest) (cpluginv1.SourceStartResponse, error) { +func (a *sourcePluginAdapter) Start(ctx context.Context, req cplugin.SourceStartRequest) (cplugin.SourceStartResponse, error) { err := a.state.DoWithLock(ctx, internal.DoWithLockOptions{ ExpectedStates: []internal.ConnectorState{internal.StateConfigured}, StateBefore: internal.StateStarting, @@ -197,10 +199,10 @@ func (a *sourcePluginAdapter) Start(ctx context.Context, req cpluginv1.SourceSta return a.impl.Open(ctxOpen, req.Position) }) - return cpluginv1.SourceStartResponse{}, err + return cplugin.SourceStartResponse{}, err } -func (a *sourcePluginAdapter) Run(ctx context.Context, stream cpluginv1.SourceRunStream) (err error) { +func (a *sourcePluginAdapter) Run(ctx context.Context, stream cplugin.SourceRunStream) (err error) { err = a.state.DoWithLock(ctx, internal.DoWithLockOptions{ ExpectedStates: []internal.ConnectorState{internal.StateStarted}, StateBefore: internal.StateInitiatingRun, @@ -216,10 +218,10 @@ func (a *sourcePluginAdapter) Run(ctx context.Context, stream cpluginv1.SourceRu t.Go(func() error { defer close(a.readDone) - return a.runRead(readCtx, stream) + return a.runRead(readCtx, stream.Server()) }) t.Go(func() error { - return a.runAck(ctx, stream) + return a.runAck(ctx, stream.Server()) }) return nil }) @@ -239,7 +241,7 @@ func (a *sourcePluginAdapter) Run(ctx context.Context, stream cpluginv1.SourceRu return a.t.Err() } -func (a *sourcePluginAdapter) runRead(ctx context.Context, stream cpluginv1.SourceRunStream) error { +func (a *sourcePluginAdapter) runRead(ctx context.Context, stream cplugin.SourceRunStreamServer) error { // TODO make backoff params configurable (https://github.com/ConduitIO/conduit/issues/184) b := &backoff.Backoff{ Factor: 2, @@ -266,7 +268,7 @@ func (a *sourcePluginAdapter) runRead(ctx context.Context, stream cpluginv1.Sour return fmt.Errorf("read plugin error: %w", err) } - err = stream.Send(cpluginv1.SourceRunResponse{Record: a.convertRecord(r)}) + err = stream.Send(cplugin.SourceRunResponse{Records: []opencdc.Record{r}}) if err != nil { return fmt.Errorf("read stream error: %w", err) } @@ -277,24 +279,27 @@ func (a *sourcePluginAdapter) runRead(ctx context.Context, stream cpluginv1.Sour } } -func (a *sourcePluginAdapter) runAck(ctx context.Context, stream cpluginv1.SourceRunStream) error { +func (a *sourcePluginAdapter) runAck(ctx context.Context, stream cplugin.SourceRunStreamServer) error { for { - req, err := stream.Recv() + batch, err := stream.Recv() if err != nil { if err == io.EOF { return nil // stream is closed, not an error } return fmt.Errorf("ack stream error: %w", err) } - err = a.impl.Ack(ctx, req.AckPosition) - // implementing Ack is optional - if err != nil && !errors.Is(err, ErrUnimplemented) { - return fmt.Errorf("ack plugin error: %w", err) + + for _, ack := range batch.AckPositions { + err = a.impl.Ack(ctx, ack) + // implementing Ack is optional + if err != nil && !errors.Is(err, ErrUnimplemented) { + return fmt.Errorf("ack plugin error: %w", err) + } } } } -func (a *sourcePluginAdapter) Stop(ctx context.Context, _ cpluginv1.SourceStopRequest) (cpluginv1.SourceStopResponse, error) { +func (a *sourcePluginAdapter) Stop(ctx context.Context, _ cplugin.SourceStopRequest) (cplugin.SourceStopResponse, error) { ctx, cancel := context.WithTimeout(ctx, stopTimeout) defer cancel() @@ -318,7 +323,7 @@ func (a *sourcePluginAdapter) Stop(ctx context.Context, _ cpluginv1.SourceStopRe return nil }) if err != nil { - return cpluginv1.SourceStopResponse{}, fmt.Errorf("failed to stop connector: %w", err) + return cplugin.SourceStopResponse{}, fmt.Errorf("failed to stop connector: %w", err) } // wait for read to actually stop running with a timeout, in case the @@ -326,15 +331,15 @@ func (a *sourcePluginAdapter) Stop(ctx context.Context, _ cpluginv1.SourceStopRe _, _, err = cchan.ChanOut[struct{}](a.readDone).Recv(ctx) if err != nil { Logger(ctx).Warn().Err(err).Msg("failed to wait for Read to stop running") - return cpluginv1.SourceStopResponse{}, fmt.Errorf("failed to stop connector: %w", err) + return cplugin.SourceStopResponse{}, fmt.Errorf("failed to stop connector: %w", err) } - return cpluginv1.SourceStopResponse{ + return cplugin.SourceStopResponse{ LastPosition: a.lastPosition, }, nil } -func (a *sourcePluginAdapter) Teardown(ctx context.Context, _ cpluginv1.SourceTeardownRequest) (cpluginv1.SourceTeardownResponse, error) { +func (a *sourcePluginAdapter) Teardown(ctx context.Context, _ cplugin.SourceTeardownRequest) (cplugin.SourceTeardownResponse, error) { err := a.state.DoWithLock(ctx, internal.DoWithLockOptions{ ExpectedStates: nil, // Teardown can be called from any state StateBefore: internal.StateTearingDown, @@ -370,17 +375,17 @@ func (a *sourcePluginAdapter) Teardown(ctx context.Context, _ cpluginv1.SourceTe return err }) - return cpluginv1.SourceTeardownResponse{}, err + return cplugin.SourceTeardownResponse{}, err } -func (a *sourcePluginAdapter) LifecycleOnCreated(ctx context.Context, req cpluginv1.SourceLifecycleOnCreatedRequest) (cpluginv1.SourceLifecycleOnCreatedResponse, error) { - return cpluginv1.SourceLifecycleOnCreatedResponse{}, a.impl.LifecycleOnCreated(ctx, req.Config) +func (a *sourcePluginAdapter) LifecycleOnCreated(ctx context.Context, req cplugin.SourceLifecycleOnCreatedRequest) (cplugin.SourceLifecycleOnCreatedResponse, error) { + return cplugin.SourceLifecycleOnCreatedResponse{}, a.impl.LifecycleOnCreated(ctx, req.Config) } -func (a *sourcePluginAdapter) LifecycleOnUpdated(ctx context.Context, req cpluginv1.SourceLifecycleOnUpdatedRequest) (cpluginv1.SourceLifecycleOnUpdatedResponse, error) { - return cpluginv1.SourceLifecycleOnUpdatedResponse{}, a.impl.LifecycleOnUpdated(ctx, req.ConfigBefore, req.ConfigAfter) +func (a *sourcePluginAdapter) LifecycleOnUpdated(ctx context.Context, req cplugin.SourceLifecycleOnUpdatedRequest) (cplugin.SourceLifecycleOnUpdatedResponse, error) { + return cplugin.SourceLifecycleOnUpdatedResponse{}, a.impl.LifecycleOnUpdated(ctx, req.ConfigBefore, req.ConfigAfter) } -func (a *sourcePluginAdapter) LifecycleOnDeleted(ctx context.Context, req cpluginv1.SourceLifecycleOnDeletedRequest) (cpluginv1.SourceLifecycleOnDeletedResponse, error) { - return cpluginv1.SourceLifecycleOnDeletedResponse{}, a.impl.LifecycleOnDeleted(ctx, req.Config) +func (a *sourcePluginAdapter) LifecycleOnDeleted(ctx context.Context, req cplugin.SourceLifecycleOnDeletedRequest) (cplugin.SourceLifecycleOnDeletedResponse, error) { + return cplugin.SourceLifecycleOnDeletedResponse{}, a.impl.LifecycleOnDeleted(ctx, req.Config) } // waitForRun returns once the Run function returns or the context gets @@ -396,59 +401,27 @@ func (a *sourcePluginAdapter) waitForRun(ctx context.Context, timeout time.Durat ) } -func (a *sourcePluginAdapter) convertRecord(r Record) cpluginv1.Record { - return cpluginv1.Record{ - Position: r.Position, - Operation: cpluginv1.Operation(r.Operation), - Metadata: r.Metadata, - Key: a.convertData(r.Key), - Payload: a.convertChange(r.Payload), - } -} - -func (a *sourcePluginAdapter) convertChange(c Change) cpluginv1.Change { - return cpluginv1.Change{ - Before: a.convertData(c.Before), - After: a.convertData(c.After), - } -} - -func (a *sourcePluginAdapter) convertData(d Data) cpluginv1.Data { - if d == nil { - return nil - } - - switch v := d.(type) { - case RawData: - return cpluginv1.RawData(v) - case StructuredData: - return cpluginv1.StructuredData(v) - default: - panic("unknown data type") - } -} - // SourceUtil provides utility methods for implementing a source. Use it by // calling Util.Source.*. type SourceUtil struct{} // NewRecordCreate can be used to instantiate a record with OperationCreate. func (SourceUtil) NewRecordCreate( - position Position, - metadata Metadata, - key Data, - payload Data, -) Record { + position opencdc.Position, + metadata opencdc.Metadata, + key opencdc.Data, + payload opencdc.Data, +) opencdc.Record { if metadata == nil { metadata = make(map[string]string) } metadata.SetReadAt(time.Now()) - return Record{ + return opencdc.Record{ Position: position, - Operation: OperationCreate, + Operation: opencdc.OperationCreate, Metadata: metadata, Key: key, - Payload: Change{ + Payload: opencdc.Change{ After: payload, }, } @@ -456,21 +429,21 @@ func (SourceUtil) NewRecordCreate( // NewRecordSnapshot can be used to instantiate a record with OperationSnapshot. func (SourceUtil) NewRecordSnapshot( - position Position, - metadata Metadata, - key Data, - payload Data, -) Record { + position opencdc.Position, + metadata opencdc.Metadata, + key opencdc.Data, + payload opencdc.Data, +) opencdc.Record { if metadata == nil { metadata = make(map[string]string) } metadata.SetReadAt(time.Now()) - return Record{ + return opencdc.Record{ Position: position, - Operation: OperationSnapshot, + Operation: opencdc.OperationSnapshot, Metadata: metadata, Key: key, - Payload: Change{ + Payload: opencdc.Change{ After: payload, }, } @@ -478,22 +451,22 @@ func (SourceUtil) NewRecordSnapshot( // NewRecordUpdate can be used to instantiate a record with OperationUpdate. func (SourceUtil) NewRecordUpdate( - position Position, - metadata Metadata, - key Data, - payloadBefore Data, - payloadAfter Data, -) Record { + position opencdc.Position, + metadata opencdc.Metadata, + key opencdc.Data, + payloadBefore opencdc.Data, + payloadAfter opencdc.Data, +) opencdc.Record { if metadata == nil { metadata = make(map[string]string) } metadata.SetReadAt(time.Now()) - return Record{ + return opencdc.Record{ Position: position, - Operation: OperationUpdate, + Operation: opencdc.OperationUpdate, Metadata: metadata, Key: key, - Payload: Change{ + Payload: opencdc.Change{ Before: payloadBefore, After: payloadAfter, }, @@ -502,17 +475,17 @@ func (SourceUtil) NewRecordUpdate( // NewRecordDelete can be used to instantiate a record with OperationDelete. func (SourceUtil) NewRecordDelete( - position Position, - metadata Metadata, - key Data, -) Record { + position opencdc.Position, + metadata opencdc.Metadata, + key opencdc.Data, +) opencdc.Record { if metadata == nil { metadata = make(map[string]string) } metadata.SetReadAt(time.Now()) - return Record{ + return opencdc.Record{ Position: position, - Operation: OperationDelete, + Operation: opencdc.OperationDelete, Metadata: metadata, Key: key, } diff --git a/source_test.go b/source_test.go index ebd432d1..1bfca565 100644 --- a/source_test.go +++ b/source_test.go @@ -17,13 +17,12 @@ package sdk import ( "context" "errors" - "fmt" "io" "testing" "time" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" - cpluginv1mock "github.com/conduitio/conduit-connector-protocol/cpluginv1/mock" + "github.com/conduitio/conduit-commons/opencdc" + "github.com/conduitio/conduit-connector-protocol/cplugin" "github.com/conduitio/conduit-connector-sdk/internal" "github.com/conduitio/conduit-connector-sdk/internal/cchan" "github.com/matryer/is" @@ -40,14 +39,14 @@ func TestSourcePluginAdapter_Start_OpenContext(t *testing.T) { srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured var gotCtx context.Context - src.EXPECT().Open(gomock.Any(), Position(nil)). - DoAndReturn(func(ctx context.Context, _ Position) error { + src.EXPECT().Open(gomock.Any(), opencdc.Position(nil)). + DoAndReturn(func(ctx context.Context, _ opencdc.Position) error { gotCtx = ctx // assign to gotCtx so it can be inspected return ctx.Err() }) ctx, cancel := context.WithCancel(context.Background()) - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{}) + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{}) is.NoErr(err) is.NoErr(gotCtx.Err()) // expected context to be open @@ -65,8 +64,8 @@ func TestSourcePluginAdapter_Start_ClosedContext(t *testing.T) { srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured var gotCtx context.Context - src.EXPECT().Open(gomock.Any(), Position(nil)). - DoAndReturn(func(ctx context.Context, _ Position) error { + src.EXPECT().Open(gomock.Any(), opencdc.Position(nil)). + DoAndReturn(func(ctx context.Context, _ opencdc.Position) error { gotCtx = ctx // assign to gotCtx so it can be inspected select { case <-ctx.Done(): @@ -79,7 +78,7 @@ func TestSourcePluginAdapter_Start_ClosedContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{}) + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{}) is.True(err != nil) is.Equal(err, ctx.Err()) is.Equal(gotCtx.Err(), context.Canceled) @@ -94,8 +93,8 @@ func TestSourcePluginAdapter_Start_Logger(t *testing.T) { srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured wantLogger := zerolog.New(zerolog.NewTestWriter(t)) - src.EXPECT().Open(gomock.Any(), Position(nil)). - DoAndReturn(func(ctx context.Context, _ Position) error { + src.EXPECT().Open(gomock.Any(), opencdc.Position(nil)). + DoAndReturn(func(ctx context.Context, _ opencdc.Position) error { gotLogger := Logger(ctx) is.True(gotLogger != nil) is.Equal(*gotLogger, wantLogger) @@ -104,7 +103,7 @@ func TestSourcePluginAdapter_Start_Logger(t *testing.T) { ctx := wantLogger.WithContext(context.Background()) - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{}) + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{}) is.NoErr(err) } @@ -116,21 +115,21 @@ func TestSourcePluginAdapter_Run(t *testing.T) { srcPlugin := NewSourcePlugin(src).(*sourcePluginAdapter) srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured - want := Record{ - Position: Position("foo"), - Operation: OperationCreate, + want := opencdc.Record{ + Position: opencdc.Position("foo"), + Operation: opencdc.OperationCreate, Metadata: map[string]string{"foo": "bar"}, - Key: RawData("bar"), - Payload: Change{ + Key: opencdc.RawData("bar"), + Payload: opencdc.Change{ Before: nil, // create has no before - After: StructuredData{ + After: opencdc.StructuredData{ "x": "y", "z": 3, }, }, } wantLast := want - wantLast.Position = Position("bar") + wantLast.Position = opencdc.Position("bar") recordCount := 5 @@ -139,12 +138,12 @@ func TestSourcePluginAdapter_Run(t *testing.T) { // first produce "normal" records, then produce last record, then return ErrBackoffRetry r1 := src.EXPECT().Read(gomock.Any()).Return(want, nil).Times(recordCount - 1) r2 := src.EXPECT().Read(gomock.Any()).Return(wantLast, nil).After(r1) - src.EXPECT().Read(gomock.Any()).Return(Record{}, ErrBackoffRetry).After(r2) - - stream, reqStream, respStream := newSourceRunStreamMock(ctrl) + src.EXPECT().Read(gomock.Any()).Return(opencdc.Record{}, ErrBackoffRetry).After(r2) ctx := context.Background() - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{Position: nil}) + stream := NewInMemorySourceRunStream(ctx) + + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{Position: nil}) is.NoErr(err) runDone := make(chan struct{}) @@ -154,53 +153,34 @@ func TestSourcePluginAdapter_Run(t *testing.T) { is.NoErr(err) }() + clientStream := stream.Client() for i := 0; i < recordCount-1; i++ { - resp := <-respStream - is.Equal(resp, cpluginv1.SourceRunResponse{ - Record: cpluginv1.Record{ - Position: want.Position, - Operation: cpluginv1.Operation(want.Operation), - Metadata: want.Metadata, - Key: cpluginv1.RawData(want.Key.(RawData)), - Payload: cpluginv1.Change{ - Before: nil, // create has no before - After: cpluginv1.StructuredData(want.Payload.After.(StructuredData)), - }, - }, - }) + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.SourceRunResponse{Records: []opencdc.Record{want}}) } // fetch last record - resp := <-respStream - is.Equal(resp, cpluginv1.SourceRunResponse{ - Record: cpluginv1.Record{ - Position: wantLast.Position, - Operation: cpluginv1.Operation(want.Operation), - Metadata: want.Metadata, - Key: cpluginv1.RawData(want.Key.(RawData)), - Payload: cpluginv1.Change{ - Before: nil, // create has no before - After: cpluginv1.StructuredData(want.Payload.After.(StructuredData)), - }, - }, - }) + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.SourceRunResponse{Records: []opencdc.Record{wantLast}}) - stopResp, err := srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + stopResp, err := srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.NoErr(err) - is.Equal([]byte(wantLast.Position), stopResp.LastPosition) // unexpected last position + is.Equal(wantLast.Position, stopResp.LastPosition) // unexpected last position // after stop the source should stop producing new records, but it will // wait for acks coming back, let's send back all acks but last one src.EXPECT().Ack(gomock.Any(), want.Position).Times(recordCount - 1) for i := 0; i < recordCount-1; i++ { - reqStream <- cpluginv1.SourceRunRequest{AckPosition: want.Position} + err = clientStream.Send(cplugin.SourceRunRequest{AckPositions: []opencdc.Position{want.Position}}) + is.NoErr(err) } // close stream - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit <-runDone @@ -221,8 +201,8 @@ func TestSourcePluginAdapter_Run_Stuck(t *testing.T) { srcPlugin := NewSourcePlugin(src).(*sourcePluginAdapter) srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured - want := Record{ - Position: Position("foo"), + want := opencdc.Record{ + Position: opencdc.Position("foo"), } src.EXPECT().Open(gomock.Any(), nil).Return(nil) @@ -233,10 +213,10 @@ func TestSourcePluginAdapter_Run_Stuck(t *testing.T) { <-make(chan struct{}) // block forever and ever }).After(r1) - stream, _, respStream := newSourceRunStreamMock(ctrl) - ctx := context.Background() - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{Position: nil}) + stream := NewInMemorySourceRunStream(ctx) + + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{Position: nil}) is.NoErr(err) runDone := make(chan struct{}) @@ -246,21 +226,19 @@ func TestSourcePluginAdapter_Run_Stuck(t *testing.T) { is.Equal("forceful teardown", err.Error()) }() - resp := <-respStream - is.Equal(resp, cpluginv1.SourceRunResponse{ - Record: cpluginv1.Record{ - Position: want.Position, - }, - }) + clientStream := stream.Client() + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.SourceRunResponse{Records: []opencdc.Record{want}}) // after this the connector starts blocking, we try to trigger a stop - stopResp, err := srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + stopResp, err := srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.True(errors.Is(err, context.DeadlineExceeded)) is.Equal(nil, stopResp.LastPosition) // unexpected last position // the connector is still blocking, teardown should detach the goroutine src.EXPECT().Teardown(gomock.Any()).Return(nil) - _, err = srcPlugin.Teardown(ctx, cpluginv1.SourceTeardownRequest{}) + _, err = srcPlugin.Teardown(ctx, cplugin.SourceTeardownRequest{}) is.True(errors.Is(err, context.DeadlineExceeded)) // wait for Run to exit, teardown killed it @@ -280,25 +258,25 @@ func TestSourcePluginAdapter_Stop_WaitsForRun(t *testing.T) { srcPlugin := NewSourcePlugin(src).(*sourcePluginAdapter) srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured - want := Record{ - Position: Position("foo"), + want := opencdc.Record{ + Position: opencdc.Position("foo"), } src.EXPECT().Open(gomock.Any(), nil).Return(nil) // produce one record, then return ErrBackoffRetry r1 := src.EXPECT().Read(gomock.Any()).Return(want, nil) - src.EXPECT().Read(gomock.Any()).Return(Record{}, ErrBackoffRetry).After(r1) + src.EXPECT().Read(gomock.Any()).Return(opencdc.Record{}, ErrBackoffRetry).After(r1) - stream, reqStream, respStream := newSourceRunStreamMock(ctrl) + ctx := context.Background() + stream := NewInMemorySourceRunStream(ctx) // Start connector now - ctx := context.Background() - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{Position: nil}) + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{Position: nil}) is.NoErr(err) // Run was not triggered yet, but we try to stop - stopResp, err := srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + stopResp, err := srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.True(errors.Is(err, context.DeadlineExceeded)) is.Equal(nil, stopResp.LastPosition) // unexpected last position @@ -310,26 +288,23 @@ func TestSourcePluginAdapter_Stop_WaitsForRun(t *testing.T) { }() // Stop should still be blocked because there is a pending record that was not read yet - stopResp, err = srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + stopResp, err = srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.True(errors.Is(err, context.DeadlineExceeded)) is.Equal(nil, stopResp.LastPosition) // unexpected last position // fetch produced record - resp := <-respStream - is.Equal(resp, cpluginv1.SourceRunResponse{ - Record: cpluginv1.Record{ - Position: want.Position, - }, - }) + clientStream := stream.Client() + resp, err := clientStream.Recv() + is.NoErr(err) + is.Equal(resp, cplugin.SourceRunResponse{Records: []opencdc.Record{want}}) // after this the connector can be stopped - stopResp, err = srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + stopResp, err = srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.NoErr(err) - is.Equal([]byte(want.Position), stopResp.LastPosition) // unexpected last position + is.Equal(want.Position, stopResp.LastPosition) // unexpected last position // close stream - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Run to exit _, _, err = cchan.ChanOut[struct{}](runDone).RecvTimeout(ctx, time.Second) @@ -345,13 +320,13 @@ func TestSourcePluginAdapter_Teardown(t *testing.T) { srcPlugin.state.Set(internal.StateConfigured) // Open expects state Configured src.EXPECT().Open(gomock.Any(), nil).Return(nil) - r1 := src.EXPECT().Read(gomock.Any()).Return(Record{}, nil) - src.EXPECT().Read(gomock.Any()).Return(Record{}, ErrBackoffRetry).After(r1) - - stream, reqStream, respStream := newSourceRunStreamMock(ctrl) + r1 := src.EXPECT().Read(gomock.Any()).Return(opencdc.Record{}, nil) + src.EXPECT().Read(gomock.Any()).Return(opencdc.Record{}, ErrBackoffRetry).After(r1) ctx := context.Background() - _, err := srcPlugin.Start(ctx, cpluginv1.SourceStartRequest{Position: nil}) + stream := NewInMemorySourceRunStream(ctx) + + _, err := srcPlugin.Start(ctx, cplugin.SourceStartRequest{Position: nil}) is.NoErr(err) runDone := make(chan struct{}) @@ -362,17 +337,19 @@ func TestSourcePluginAdapter_Teardown(t *testing.T) { }() // fetch one record from stream to ensure Run started - <-respStream + clientStream := stream.Client() + _, err = clientStream.Recv() + is.NoErr(err) // stop the record producing goroutine - _, err = srcPlugin.Stop(ctx, cpluginv1.SourceStopRequest{}) + _, err = srcPlugin.Stop(ctx, cplugin.SourceStopRequest{}) is.NoErr(err) // teardown should block until the stream is closed and all acks were received teardownDone := make(chan struct{}) go func() { defer close(teardownDone) - _, err := srcPlugin.Teardown(ctx, cpluginv1.SourceTeardownRequest{}) + _, err := srcPlugin.Teardown(ctx, cplugin.SourceTeardownRequest{}) is.NoErr(err) }() @@ -385,8 +362,7 @@ func TestSourcePluginAdapter_Teardown(t *testing.T) { // close stream and unblock teardown src.EXPECT().Teardown(gomock.Any()).Return(nil) - close(reqStream) - close(respStream) + stream.Close(io.EOF) // wait for Teardown and Run to exit <-teardownDone @@ -404,7 +380,7 @@ func TestSourcePluginAdapter_LifecycleOnCreated(t *testing.T) { want := map[string]string{"foo": "bar"} src.EXPECT().LifecycleOnCreated(ctx, want).Return(nil) - req := cpluginv1.SourceLifecycleOnCreatedRequest{Config: want} + req := cplugin.SourceLifecycleOnCreatedRequest{Config: want} _, err := srcPlugin.LifecycleOnCreated(ctx, req) is.NoErr(err) } @@ -421,7 +397,7 @@ func TestSourcePluginAdapter_LifecycleOnUpdated(t *testing.T) { wantAfter := map[string]string{"foo": "baz"} src.EXPECT().LifecycleOnUpdated(ctx, wantBefore, wantAfter).Return(nil) - req := cpluginv1.SourceLifecycleOnUpdatedRequest{ + req := cplugin.SourceLifecycleOnUpdatedRequest{ ConfigBefore: wantBefore, ConfigAfter: wantAfter, } @@ -440,46 +416,7 @@ func TestSourcePluginAdapter_LifecycleOnDeleted(t *testing.T) { want := map[string]string{"foo": "bar"} src.EXPECT().LifecycleOnDeleted(ctx, want).Return(nil) - req := cpluginv1.SourceLifecycleOnDeletedRequest{Config: want} + req := cplugin.SourceLifecycleOnDeletedRequest{Config: want} _, err := srcPlugin.LifecycleOnDeleted(ctx, req) is.NoErr(err) } - -func newSourceRunStreamMock( - ctrl *gomock.Controller, -) ( - *cpluginv1mock.SourceRunStream, - chan cpluginv1.SourceRunRequest, - chan cpluginv1.SourceRunResponse, -) { - stream := cpluginv1mock.NewSourceRunStream(ctrl) - - reqStream := make(chan cpluginv1.SourceRunRequest) - respStream := make(chan cpluginv1.SourceRunResponse) - - stream.EXPECT().Send(gomock.Any()). - DoAndReturn(func(resp cpluginv1.SourceRunResponse) (err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - err, ok = r.(error) - if !ok { - err = fmt.Errorf("%+v", r) - } - } - }() - respStream <- resp - return nil - }).AnyTimes() - - stream.EXPECT().Recv(). - DoAndReturn(func() (cpluginv1.SourceRunRequest, error) { - req, ok := <-reqStream - if !ok { - return cpluginv1.SourceRunRequest{}, io.EOF - } - return req, nil - }).AnyTimes() - - return stream, reqStream, respStream -} diff --git a/specifier.go b/specifier.go index 7276627c..896ae05e 100644 --- a/specifier.go +++ b/specifier.go @@ -18,7 +18,7 @@ import ( "context" "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" + "github.com/conduitio/conduit-connector-protocol/cplugin" ) // Specification contains general information regarding the plugin like its name @@ -39,65 +39,9 @@ type Specification struct { Author string } -// Parameter defines a single connector parameter. -type Parameter struct { - // Default is the default value of the parameter, if any. - Default string - // Required controls if the parameter will be shown as required or optional. - // Deprecated: add ValidationRequired to Parameter.Validations instead. - Required bool - // Description holds a description of the field and how to configure it. - Description string - // Type defines the parameter data type. - Type ParameterType - // Validations slice of validations to be checked for the parameter. - Validations []Validation -} - -func (p Parameter) toConfigParameter() config.Parameter { - var validations []config.Validation - - var isRequired bool - if p.Validations != nil { - validations = make([]config.Validation, len(p.Validations)) - for i, v := range p.Validations { - val := v.configValidation() - if val.Type() == config.ValidationTypeRequired { - isRequired = true - } - validations[i] = val - } - } - if p.Required && !isRequired { - //nolint:makezero // This is done for backwards compatibility and is the expected behavior. - validations = append(validations, config.ValidationRequired{}) - } - - return config.Parameter{ - Default: p.Default, - Description: p.Description, - Type: config.ParameterType(p.Type), - Validations: validations, - } -} - -// utility type to convert a slice of Parameter to a slice of config.Parameter -type parameters map[string]Parameter - -func (p parameters) toConfigParameters() config.Parameters { - if p == nil { - return nil - } - out := make(config.Parameters, len(p)) - for k, v := range p { - out[k] = v.toConfigParameter() - } - return out -} - // NewSpecifierPlugin takes a Specification and wraps it into an adapter that -// converts it into a cpluginv1.SpecifierPlugin. -func NewSpecifierPlugin(specs Specification, source Source, dest Destination) cpluginv1.SpecifierPlugin { +// converts it into a cplugin.SpecifierPlugin. +func NewSpecifierPlugin(specs Specification, source Source, dest Destination) cplugin.SpecifierPlugin { if source == nil { // prevent nil pointer source = UnimplementedSource{} @@ -116,36 +60,18 @@ func NewSpecifierPlugin(specs Specification, source Source, dest Destination) cp type specifierPluginAdapter struct { specs Specification - sourceParams map[string]Parameter - destinationParams map[string]Parameter + sourceParams config.Parameters + destinationParams config.Parameters } -func (s *specifierPluginAdapter) Specify(context.Context, cpluginv1.SpecifierSpecifyRequest) (cpluginv1.SpecifierSpecifyResponse, error) { - return cpluginv1.SpecifierSpecifyResponse{ +func (s *specifierPluginAdapter) Specify(context.Context, cplugin.SpecifierSpecifyRequest) (cplugin.SpecifierSpecifyResponse, error) { + return cplugin.SpecifierSpecifyResponse{ Name: s.specs.Name, Summary: s.specs.Summary, Description: s.specs.Description, Version: s.specs.Version, Author: s.specs.Author, - SourceParams: s.convertParameters(s.sourceParams), - DestinationParams: s.convertParameters(s.destinationParams), + SourceParams: s.sourceParams, + DestinationParams: s.destinationParams, }, nil } - -func (s *specifierPluginAdapter) convertParameters(params map[string]Parameter) map[string]cpluginv1.SpecifierParameter { - out := make(map[string]cpluginv1.SpecifierParameter) - for k, v := range params { - out[k] = s.convertParameter(v) - } - return out -} - -func (s *specifierPluginAdapter) convertParameter(p Parameter) cpluginv1.SpecifierParameter { - return cpluginv1.SpecifierParameter{ - Default: p.Default, - Required: p.Required, - Description: p.Description, - Type: cpluginv1.ParameterType(p.Type), - Validations: convertValidations(p.Validations), - } -} diff --git a/specifier_test.go b/specifier_test.go index 42199e96..77b63464 100644 --- a/specifier_test.go +++ b/specifier_test.go @@ -18,7 +18,7 @@ import ( "context" "testing" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" + "github.com/conduitio/conduit-connector-protocol/cplugin" "github.com/matryer/is" ) @@ -26,7 +26,7 @@ func TestSpecifier_NilSource(t *testing.T) { is := is.New(t) // ensure that having a connector without a source still works p := NewSpecifierPlugin(Specification{}, nil, UnimplementedDestination{}) - _, err := p.Specify(context.Background(), cpluginv1.SpecifierSpecifyRequest{}) + _, err := p.Specify(context.Background(), cplugin.SpecifierSpecifyRequest{}) is.NoErr(err) } @@ -34,6 +34,6 @@ func TestSpecifier_NilDestination(t *testing.T) { is := is.New(t) // ensure that having a connector without a destination still works p := NewSpecifierPlugin(Specification{}, UnimplementedSource{}, nil) - _, err := p.Specify(context.Background(), cpluginv1.SpecifierSpecifyRequest{}) + _, err := p.Specify(context.Background(), cplugin.SpecifierSpecifyRequest{}) is.NoErr(err) } diff --git a/stream_test.go b/stream_test.go new file mode 100644 index 00000000..02705155 --- /dev/null +++ b/stream_test.go @@ -0,0 +1,152 @@ +// Copyright © 2024 Meroxa, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "context" + "io" + "sync" + + "github.com/conduitio/conduit-connector-protocol/cplugin" +) + +type InMemoryDestinationRunStream inMemoryStream[cplugin.DestinationRunRequest, cplugin.DestinationRunResponse] + +func NewInMemoryDestinationRunStream(ctx context.Context) *InMemoryDestinationRunStream { + return &InMemoryDestinationRunStream{ + ctx: ctx, + reqChan: make(chan cplugin.DestinationRunRequest), + respChan: make(chan cplugin.DestinationRunResponse), + stopChan: make(chan struct{}), + } +} + +func (s *InMemoryDestinationRunStream) Client() cplugin.DestinationRunStreamClient { + return (*inMemoryStreamClient[cplugin.DestinationRunRequest, cplugin.DestinationRunResponse])(s) +} +func (s *InMemoryDestinationRunStream) Server() cplugin.DestinationRunStreamServer { + return (*inMemoryStreamServer[cplugin.DestinationRunRequest, cplugin.DestinationRunResponse])(s) +} +func (s *InMemoryDestinationRunStream) Close(reason error) bool { + return ((*inMemoryStream[cplugin.DestinationRunRequest, cplugin.DestinationRunResponse])(s)).Close(reason) +} + +type InMemorySourceRunStream inMemoryStream[cplugin.SourceRunRequest, cplugin.SourceRunResponse] + +func NewInMemorySourceRunStream(ctx context.Context) *InMemorySourceRunStream { + return &InMemorySourceRunStream{ + ctx: ctx, + reqChan: make(chan cplugin.SourceRunRequest), + respChan: make(chan cplugin.SourceRunResponse), + stopChan: make(chan struct{}), + } +} + +func (s *InMemorySourceRunStream) Client() cplugin.SourceRunStreamClient { + return (*inMemoryStreamClient[cplugin.SourceRunRequest, cplugin.SourceRunResponse])(s) +} +func (s *InMemorySourceRunStream) Server() cplugin.SourceRunStreamServer { + return (*inMemoryStreamServer[cplugin.SourceRunRequest, cplugin.SourceRunResponse])(s) +} +func (s *InMemorySourceRunStream) Close(reason error) bool { + return ((*inMemoryStream[cplugin.SourceRunRequest, cplugin.SourceRunResponse])(s)).Close(reason) +} + +type inMemoryStream[REQ any, RES any] struct { + ctx context.Context + reqChan chan REQ + respChan chan RES + stopChan chan struct{} + + reason error + m sync.Mutex +} + +func (s *inMemoryStream[REQ, RES]) Close(reason error) bool { + s.m.Lock() + defer s.m.Unlock() + select { + case <-s.stopChan: + // channel already closed + return false + default: + s.reason = reason + close(s.stopChan) + return true + } +} + +// inMemoryStreamClient mimics the behavior of a gRPC client stream using channels. +// REQ represents the type sent from the client to the server, RES is the type +// sent from the server to the client. +type inMemoryStreamClient[REQ any, RES any] inMemoryStream[REQ, RES] + +func (s *inMemoryStreamClient[REQ, RES]) Send(req REQ) error { + select { + case <-s.ctx.Done(): + return s.ctx.Err() + case <-s.stopChan: + return io.EOF + case s.reqChan <- req: + return nil + } +} + +func (s *inMemoryStreamClient[REQ, RES]) Recv() (RES, error) { + select { + case <-s.ctx.Done(): + return s.emptyRes(), s.ctx.Err() + case <-s.stopChan: + return s.emptyRes(), s.reason // client receives the reason for closing + case resp := <-s.respChan: + return resp, nil + } +} +func (s *inMemoryStreamClient[REQ, RES]) emptyRes() RES { + var r RES + return r +} + +// inMemoryStreamServer mimics the behavior of a gRPC server stream using channels. +// REQ represents the type sent from the client to the server, RES is the type +// sent from the server to the client. +type inMemoryStreamServer[REQ any, RES any] inMemoryStream[REQ, RES] + +func (s *inMemoryStreamServer[REQ, RES]) Send(resp RES) error { + select { + case <-s.ctx.Done(): + return s.ctx.Err() + case <-s.stopChan: + return io.EOF + case s.respChan <- resp: + return nil + } +} + +func (s *inMemoryStreamServer[REQ, RES]) Recv() (REQ, error) { + select { + case <-s.ctx.Done(): + return s.emptyReq(), s.ctx.Err() + case <-s.stopChan: + return s.emptyReq(), io.EOF + case req := <-s.reqChan: + return req, nil + } +} + +func (s *inMemoryStreamServer[REQ, RES]) emptyReq() REQ { + var r REQ + return r +} diff --git a/unimplemented.go b/unimplemented.go index 1375c4d0..62eda61e 100644 --- a/unimplemented.go +++ b/unimplemented.go @@ -17,13 +17,16 @@ package sdk import ( "context" "fmt" + + "github.com/conduitio/conduit-commons/config" + "github.com/conduitio/conduit-commons/opencdc" ) // UnimplementedDestination should be embedded to have forward compatible implementations. type UnimplementedDestination struct{} // Parameters needs to be overridden in the actual implementation. -func (UnimplementedDestination) Parameters() map[string]Parameter { +func (UnimplementedDestination) Parameters() config.Parameters { return nil } @@ -38,7 +41,7 @@ func (UnimplementedDestination) Open(context.Context) error { } // Write needs to be overridden in the actual implementation. -func (UnimplementedDestination) Write(context.Context, []Record) (int, error) { +func (UnimplementedDestination) Write(context.Context, []opencdc.Record) (int, error) { return 0, fmt.Errorf("action \"Write\": %w", ErrUnimplemented) } @@ -68,7 +71,7 @@ func (UnimplementedDestination) mustEmbedUnimplementedDestination() {} type UnimplementedSource struct{} // Parameters needs to be overridden in the actual implementation. -func (UnimplementedSource) Parameters() map[string]Parameter { +func (UnimplementedSource) Parameters() config.Parameters { return nil } @@ -78,18 +81,18 @@ func (UnimplementedSource) Configure(context.Context, map[string]string) error { } // Open needs to be overridden in the actual implementation. -func (UnimplementedSource) Open(context.Context, Position) error { +func (UnimplementedSource) Open(context.Context, opencdc.Position) error { return fmt.Errorf("action \"Open\": %w", ErrUnimplemented) } // Read needs to be overridden in the actual implementation. -func (UnimplementedSource) Read(context.Context) (Record, error) { - return Record{}, fmt.Errorf("action \"Read\": %w", ErrUnimplemented) +func (UnimplementedSource) Read(context.Context) (opencdc.Record, error) { + return opencdc.Record{}, fmt.Errorf("action \"Read\": %w", ErrUnimplemented) } // Ack should be overridden if acks need to be forwarded to the source, // otherwise it is optional. -func (UnimplementedSource) Ack(context.Context, Position) error { +func (UnimplementedSource) Ack(context.Context, opencdc.Position) error { return fmt.Errorf("action \"Ack\": %w", ErrUnimplemented) } diff --git a/util.go b/util.go index 981fa39a..f49d237c 100644 --- a/util.go +++ b/util.go @@ -35,8 +35,8 @@ var Util = struct { ParseConfig: parseConfig, } -func mergeParameters(p1 map[string]Parameter, p2 map[string]Parameter) map[string]Parameter { - params := make(map[string]Parameter, len(p1)+len(p2)) +func mergeParameters(p1 config.Parameters, p2 config.Parameters) config.Parameters { + params := make(config.Parameters, len(p1)+len(p2)) for k, v := range p1 { params[k] = v } diff --git a/validation.go b/validation.go deleted file mode 100644 index 19046857..00000000 --- a/validation.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "regexp" - - "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" -) - -var ( - ErrUnrecognizedParameter = config.ErrUnrecognizedParameter - ErrInvalidParameterValue = config.ErrInvalidParameterValue - ErrInvalidParameterType = config.ErrInvalidParameterType - ErrInvalidValidationType = config.ErrInvalidValidationType - ErrRequiredParameterMissing = config.ErrRequiredParameterMissing - - ErrLessThanValidationFail = config.ErrLessThanValidationFail - ErrGreaterThanValidationFail = config.ErrGreaterThanValidationFail - ErrInclusionValidationFail = config.ErrInclusionValidationFail - ErrExclusionValidationFail = config.ErrExclusionValidationFail - ErrRegexValidationFail = config.ErrRegexValidationFail -) - -const ( - ParameterTypeString ParameterType = iota + 1 - ParameterTypeInt - ParameterTypeFloat - ParameterTypeBool - ParameterTypeFile - ParameterTypeDuration -) - -type ParameterType config.ParameterType - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - var cTypes [1]struct{} - _ = cTypes[int(ParameterTypeString)-int(cpluginv1.ParameterTypeString)] - _ = cTypes[int(ParameterTypeInt)-int(cpluginv1.ParameterTypeInt)] - _ = cTypes[int(ParameterTypeFloat)-int(cpluginv1.ParameterTypeFloat)] - _ = cTypes[int(ParameterTypeBool)-int(cpluginv1.ParameterTypeBool)] - _ = cTypes[int(ParameterTypeFile)-int(cpluginv1.ParameterTypeFile)] - _ = cTypes[int(ParameterTypeDuration)-int(cpluginv1.ParameterTypeDuration)] - - _ = cTypes[int(ParameterTypeString)-int(config.ParameterTypeString)] - _ = cTypes[int(ParameterTypeInt)-int(config.ParameterTypeInt)] - _ = cTypes[int(ParameterTypeFloat)-int(config.ParameterTypeFloat)] - _ = cTypes[int(ParameterTypeBool)-int(config.ParameterTypeBool)] - _ = cTypes[int(ParameterTypeFile)-int(config.ParameterTypeFile)] - _ = cTypes[int(ParameterTypeDuration)-int(config.ParameterTypeDuration)] - - _ = cTypes[int(config.ValidationTypeRequired)-int(cpluginv1.ValidationTypeRequired)] - _ = cTypes[int(config.ValidationTypeGreaterThan)-int(cpluginv1.ValidationTypeGreaterThan)] - _ = cTypes[int(config.ValidationTypeLessThan)-int(cpluginv1.ValidationTypeLessThan)] - _ = cTypes[int(config.ValidationTypeInclusion)-int(cpluginv1.ValidationTypeInclusion)] - _ = cTypes[int(config.ValidationTypeExclusion)-int(cpluginv1.ValidationTypeExclusion)] - _ = cTypes[int(config.ValidationTypeRegex)-int(cpluginv1.ValidationTypeRegex)] -} - -type Validation interface { - configValidation() config.Validation -} - -type ValidationRequired struct{} - -func (v ValidationRequired) configValidation() config.Validation { - return config.ValidationRequired(v) -} - -type ValidationLessThan struct { - Value float64 -} - -func (v ValidationLessThan) configValidation() config.Validation { - return config.ValidationLessThan{V: v.Value} -} - -type ValidationGreaterThan struct { - Value float64 -} - -func (v ValidationGreaterThan) configValidation() config.Validation { - return config.ValidationGreaterThan{V: v.Value} -} - -type ValidationInclusion struct { - List []string -} - -func (v ValidationInclusion) configValidation() config.Validation { - return config.ValidationInclusion{List: v.List} -} - -type ValidationExclusion struct { - List []string -} - -func (v ValidationExclusion) configValidation() config.Validation { - return config.ValidationExclusion{List: v.List} -} - -type ValidationRegex struct { - Regex *regexp.Regexp -} - -func (v ValidationRegex) configValidation() config.Validation { - return config.ValidationRegex{Regex: v.Regex} -} - -func convertValidations(validations []Validation) []cpluginv1.ParameterValidation { - if validations == nil { - return nil - } - out := make([]cpluginv1.ParameterValidation, len(validations)) - for i, v := range validations { - val := v.configValidation() - out[i] = cpluginv1.ParameterValidation{ - Type: cpluginv1.ValidationType(val.Type()), - Value: val.Value(), - } - } - return out -} diff --git a/validation_test.go b/validation_test.go deleted file mode 100644 index d9d3a55a..00000000 --- a/validation_test.go +++ /dev/null @@ -1,538 +0,0 @@ -// Copyright © 2022 Meroxa, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "errors" - "regexp" - "testing" - - "github.com/conduitio/conduit-commons/config" - "github.com/conduitio/conduit-connector-protocol/cpluginv1" - "github.com/matryer/is" -) - -func TestValidation_Param_Type(t *testing.T) { - is := is.New(t) - tests := []struct { - name string - config map[string]string - params map[string]Parameter - wantErr bool - }{ - { - name: "valid type number", - config: map[string]string{ - "param1": "3", - }, - params: map[string]Parameter{ - "param1": { - Default: "3.3", - Type: ParameterTypeFloat, - }, - }, - wantErr: false, - }, - { - name: "invalid type float", - config: map[string]string{ - "param1": "not-a-number", - }, - params: map[string]Parameter{ - "param1": { - Default: "3.3", - Type: ParameterTypeFloat, - }, - }, - wantErr: true, - }, - { - name: "valid default type float", - config: map[string]string{ - "param1": "", - }, - params: map[string]Parameter{ - "param1": { - Default: "3", - Type: ParameterTypeFloat, - }, - }, - wantErr: false, - }, - { - name: "valid type int", - config: map[string]string{ - "param1": "3", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeInt, - }, - }, - wantErr: false, - }, - { - name: "invalid type int", - config: map[string]string{ - "param1": "3.3", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeInt, - }, - }, - wantErr: true, - }, - { - name: "valid type bool", - config: map[string]string{ - "param1": "1", // 1, t, T, True, TRUE are all valid booleans - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeBool, - }, - }, - wantErr: false, - }, - { - name: "valid type bool", - config: map[string]string{ - "param1": "true", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeBool, - }, - }, - wantErr: false, - }, - { - name: "invalid type bool", - config: map[string]string{ - "param1": "not-a-bool", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeBool, - }, - }, - wantErr: true, - }, - { - name: "valid type duration", - config: map[string]string{ - "param1": "1s", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeDuration, - }, - }, - wantErr: false, - }, - { - name: "empty value is valid for all types", - config: map[string]string{ - "param1": "", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeDuration, - }, - }, - wantErr: false, - }, - { - name: "invalid type duration", - config: map[string]string{ - "param1": "not-a-duration", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeDuration, - }, - }, - wantErr: true, - }, - { - name: "valid type string", - config: map[string]string{ - "param1": "param", - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeString, - }, - }, - wantErr: false, - }, - { - name: "valid type file", - config: map[string]string{ - "param1": "some-data", // a file is a slice of bytes, so any string is valid - }, - params: map[string]Parameter{ - "param1": { - Type: ParameterTypeFile, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - params := parameters(tt.params).toConfigParameters() - - err := config.Config(tt.config). - Sanitize(). - ApplyDefaults(params). - Validate(params) - if err != nil && tt.wantErr { - is.True(errors.Is(err, ErrInvalidParameterType)) - } else if err != nil || tt.wantErr { - t.Errorf("UtilityFunc() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestValidation_Param_Value(t *testing.T) { - is := is.New(t) - - tests := []struct { - name string - config map[string]string - params map[string]Parameter - wantErr bool - err error - }{ - { - name: "required validation failed", - config: map[string]string{ - "param1": "", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - }}, - }, - wantErr: true, - err: ErrRequiredParameterMissing, - }, - { - name: "required validation pass", - config: map[string]string{ - "param1": "value", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - }}, - }, - wantErr: false, - }, - { - name: "less than validation failed", - config: map[string]string{ - "param1": "20", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationLessThan{10}, - }}, - }, - wantErr: true, - err: ErrLessThanValidationFail, - }, - { - name: "less than validation pass", - config: map[string]string{ - "param1": "0", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationLessThan{10}, - }}, - }, - wantErr: false, - }, - { - name: "greater than validation failed", - config: map[string]string{ - "param1": "0", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationGreaterThan{10}, - }}, - }, - wantErr: true, - err: ErrGreaterThanValidationFail, - }, - { - name: "greater than validation failed", - config: map[string]string{ - "param1": "20", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationGreaterThan{10}, - }}, - }, - wantErr: false, - }, - { - name: "inclusion validation failed", - config: map[string]string{ - "param1": "three", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationInclusion{[]string{"one", "two"}}, - }}, - }, - wantErr: true, - err: ErrInclusionValidationFail, - }, - { - name: "inclusion validation pass", - config: map[string]string{ - "param1": "two", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationInclusion{[]string{"one", "two"}}, - }}, - }, - wantErr: false, - }, - { - name: "exclusion validation failed", - config: map[string]string{ - "param1": "one", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationExclusion{[]string{"one", "two"}}, - }}, - }, - wantErr: true, - err: ErrExclusionValidationFail, - }, - { - name: "exclusion validation pass", - config: map[string]string{ - "param1": "three", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationExclusion{[]string{"one", "two"}}, - }}, - }, - wantErr: false, - }, - { - name: "regex validation failed", - config: map[string]string{ - "param1": "a-a", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationRegex{regexp.MustCompile("[a-z]-[1-9]")}, - }}, - }, - wantErr: true, - err: ErrRegexValidationFail, - }, - { - name: "regex validation pass", - config: map[string]string{ - "param1": "a-8", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationRequired{}, - ValidationRegex{regexp.MustCompile("[a-z]-[1-9]")}, - }}, - }, - wantErr: false, - }, - { - name: "optional validation pass", - config: map[string]string{ - "param1": "", - }, - params: map[string]Parameter{ - "param1": {Validations: []Validation{ - ValidationInclusion{[]string{"one", "two"}}, - ValidationExclusion{[]string{"three", "four"}}, - ValidationRegex{regexp.MustCompile("[a-z]")}, - ValidationGreaterThan{10}, - ValidationLessThan{20}, - }}, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - params := parameters(tt.params).toConfigParameters() - - err := config.Config(tt.config). - Sanitize(). - ApplyDefaults(params). - Validate(params) - if err != nil && tt.wantErr { - is.True(errors.Is(err, tt.err)) - } else if err != nil || tt.wantErr { - t.Errorf("UtilityFunc() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestValidation_toCPluginV1(t *testing.T) { - is := is.New(t) - validations := []Validation{ - ValidationRequired{}, - ValidationLessThan{5.10}, - ValidationGreaterThan{0}, - ValidationInclusion{[]string{"1", "2"}}, - ValidationExclusion{[]string{"3", "4"}}, - ValidationRegex{regexp.MustCompile("[a-z]*")}, - } - want := []cpluginv1.ParameterValidation{ - { - Type: cpluginv1.ValidationTypeRequired, - Value: "", - }, { - Type: cpluginv1.ValidationTypeLessThan, - Value: "5.1", - }, { - Type: cpluginv1.ValidationTypeGreaterThan, - Value: "0", - }, { - Type: cpluginv1.ValidationTypeInclusion, - Value: "1,2", - }, { - Type: cpluginv1.ValidationTypeExclusion, - Value: "3,4", - }, { - Type: cpluginv1.ValidationTypeRegex, - Value: "[a-z]*", - }, - } - got := convertValidations(validations) - is.Equal(got, want) -} - -func TestValidation_Multi_Error(t *testing.T) { - is := is.New(t) - - specParams := map[string]Parameter{ - "limit": { - Type: ParameterTypeInt, - Validations: []Validation{ - ValidationGreaterThan{0}, - ValidationRegex{regexp.MustCompile("^[0-9]")}, - }}, - "option": { - Type: ParameterTypeString, - Validations: []Validation{ - ValidationInclusion{[]string{"one", "two", "three"}}, - ValidationExclusion{[]string{"one", "five"}}, - }}, - "name": { - Type: ParameterTypeString, - Validations: []Validation{ - ValidationRequired{}, - }}, - } - cfg := map[string]string{ - "limit": "-1", - "option": "five", - } - - params := parameters(specParams).toConfigParameters() - - err := config.Config(cfg). - Sanitize(). - ApplyDefaults(params). - Validate(params) - is.True(err != nil) - - // name is missing - is.True(errors.Is(err, ErrRequiredParameterMissing)) - // option is not included in list - is.True(errors.Is(err, ErrInclusionValidationFail)) - // option is excluded from list - is.True(errors.Is(err, ErrExclusionValidationFail)) - // limit is not greater than 0 - is.True(errors.Is(err, ErrGreaterThanValidationFail)) - // limit does not match the regex pattern - is.True(errors.Is(err, ErrRegexValidationFail)) -} - -func TestValidation_initConfig(t *testing.T) { - is := is.New(t) - - specParams := map[string]Parameter{ - "param1": { - Type: ParameterTypeString, - Default: "param1", - }, - "param2": { - Type: ParameterTypeString, - Default: "param2", - }, - "param3": { - Type: ParameterTypeString, - Default: "param3", - }, - "param4": { - Type: ParameterTypeString, - Default: "param4", - }, - } - cfg := map[string]string{ - "param1": "not-default", - "param2": "", - } - - want := map[string]string{ - "param1": "not-default", - "param2": "param2", - "param3": "param3", - "param4": "param4", - } - - params := parameters(specParams).toConfigParameters() - - got := config.Config(cfg). - Sanitize(). - ApplyDefaults(params) - - is.Equal(map[string]string(got), want) -}