Merge branch 'main' into lovro/refactor-connector-plugin

.github/workflows/release.yml

@@ -62,7 +62,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5.4.0
+        uses: docker/metadata-action@v5.5.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # Makes no sense to have an image with tag v0, hence the check in the last line.

Dockerfile

@@ -26,5 +26,4 @@ EXPOSE 8080/tcp
 EXPOSE 8084/tcp
 WORKDIR /app
 COPY --from=base /app/conduit /app
-COPY --from=base /app/conduit-pipeline-check /app
 CMD ["/app/conduit"]

Makefile

@@ -9,7 +9,7 @@ VERSION=`git describe --tags --dirty`
 GO_VERSION_CHECK=`./scripts/check-go-version.sh`
 
 # The build target should stay at the top since we want it to be the default target.
-build: check-go-version pkg/web/ui/dist build-pipeline-check
+build: check-go-version pkg/web/ui/dist
 	go build -ldflags "-X 'github.com/conduitio/conduit/pkg/conduit.version=${VERSION}'" -o conduit -tags ui ./cmd/conduit/main.go
 	@echo "\nBuild complete. Enjoy using Conduit!"
 	@echo "Get started by running:"
@@ -39,9 +39,6 @@ build-server: check-go-version
 	go build -ldflags "-X 'github.com/conduitio/conduit/pkg/conduit.version=${VERSION}'" -o conduit ./cmd/conduit/main.go
 	@echo "build version: ${VERSION}"
 
-build-pipeline-check: check-go-version
-	go build -o conduit-pipeline-check ./cmd/conduit-pipeline-check/main.go
-
 run:
 	go run ./cmd/conduit/main.go
 
@@ -56,7 +53,6 @@ proto-lint:
 
 clean:
 	@rm -f conduit
-	@rm -f conduit-pipeline-check
 	@rm -rf pkg/web/ui/dist
 
 download:

README.md

@@ -191,9 +191,9 @@ each configuration option based on the following priorities:
 ## Connectors
 
 For the full list of available connectors, see
-the [Connector List](docs/connectors.md). If there's a connector that you're
-looking for that isn't available in Conduit, please file
-an [issue](https://github.com/ConduitIO/conduit/issues/new?assignees=&labels=triage&template=3-connector-request.yml&title=Connector%3A+%3Cresource%3E+%5BSource%2FDestination%5D)
+the [Connector List](https://conduit.io/docs/connectors/connector-list). If
+there's a connector that you're looking for that isn't available in Conduit,
+please file an [issue](https://github.com/ConduitIO/conduit/issues/new?assignees=&labels=triage&template=3-connector-request.yml&title=Connector%3A+%3Cresource%3E+%5BSource%2FDestination%5D)
 .
 
 Conduit loads standalone connectors at startup. The connector binaries need to

docs/connectors.md

@@ -82,51 +82,11 @@ The Conduit team and other community developers create and maintain standalone c
 
 Learn more about how you can install [standalone connectors to Conduit here](https://conduit.io/docs/connectors/installing).
 
-### Support Types
-
-- `Conduit` - These are connectors that are built by the Conduit. Any issues or problems filed on those repos will be
-  responded to by the Conduit team.
-- `Community` - A community connector is one where a developer created a connector and they're the ones supporting it
-  not the Conduit team.
-- `Legacy` - Some connectors are built using non-preferred methods. For example, Kafka Connect connectors can be used
-  on Conduit. This is considered a stop gap measure until `conduit` or `community` connectors are built.
-
-At this time, Conduit does not have any commercially supported connectors.
-
 ### Source & Destination
 
 Source means the connector has the ability to get data from an upstream data store. Destination means the connector can
 to write to a downstream data store.
 
 ### The List
 
-| Connector | Source | Destination | Support | Conduit version support |
-|-----------|-------|----|-------------|-------------|
-| [Airtable](https://github.com/conduitio-labs/conduit-connector-airtable) |WIP| | Community | WIP |
-| [Algolia](https://github.com/conduitio-labs/conduit-connector-algolia) | |✅ | Conduit | v0.3.0 |
-| [Azure Storage](https://github.com/conduitio-labs/conduit-connector-azure-storage) |✅ | | Community | v0.3.0|
-| [BigQuery](https://github.com/conduitio-labs/conduit-connector-bigquery) |✅ | | Community | v0.3.0|
-| [DB2](https://github.com/conduitio-labs/conduit-connector-db2) |WIP|WIP| Community |WIP|
-| [Elasticsearch](https://github.com/conduitio-labs/conduit-connector-elasticsearch) |✅ |✅ | Community |v0.3.0|
-| [File](https://github.com/ConduitIO/conduit-connector-file) |✅ |✅ | Conduit |v0.3.0|
-| [Firebolt](https://github.com/conduitio-labs/conduit-connector-firebolt) |WIP |WIP| Community | WIP |
-| [GCP PubSub](https://github.com/conduitio-labs/conduit-connector-gcp-pubsub) |✅ |✅| Community |WIP |
-| [Google Cloud Storage](https://github.com/conduitio-labs/conduit-connector-google-cloudstorage) |✅ | | Community | v0.3.0|
-| [Google Sheets](https://github.com/conduitio-labs/conduit-connector-google-sheets) |✅ | | Community | v0.3.0|
-| [Kafka](https://github.com/ConduitIO/conduit-connector-kafka) |✅ |✅ | Conduit |v0.3.0|
-| [Kafka Connect Wrapper](https://github.com/ConduitIO/conduit-kafka-connect-wrapper) | ✅ | ✅ | Legacy |v0.3.0|
-| [Marketo](https://github.com/conduitio-labs/conduit-connector-marketo) |✅ | | Community |v0.3.0|
-| [Materialize](https://github.com/conduitio-labs/conduit-connector-materialize) | |✅ | Community |v0.3.0|
-| [MongoDB](https://github.com/conduitio-labs/conduit-connector-mongo) |✅ |✅ | Community |v0.6.0|
-| [Nats Jetstream](https://github.com/conduitio-labs/conduit-connector-nats-jetstream) |✅ |✅ | Community |v0.3.0|
-| [Nats PubSub](https://github.com/conduitio-labs/conduit-connector-nats-pubsub) |✅|✅ | Community |v0.3.0|
-| [Oracle DB](https://github.com/conduitio-labs/conduit-connector-oracle) |WIP|WIP| Community |WIP|
-| [Postgres](https://github.com/ConduitIO/conduit-connector-postgres)   |✅ |✅ | Conduit |v0.3.0|
-| [Random Generator](https://github.com/ConduitIO/conduit-connector-generator) |✅ | | Conduit |v0.3.0|
-| [Redis](https://github.com/conduitio-labs/conduit-connector-redis) |✅|✅ | Community |v0.3.0|
-| [S3](https://github.com/ConduitIO/conduit-connector-s3) |✅ |✅ | Conduit | v0.3.0|
-| [Salesforce](https://github.com/conduitio-labs/conduit-connector-salesforce) | ✅ | | Community | v0.3.0|
-| [Snowflake](https://github.com/conduitio-labs/conduit-connector-snowflake) |✅ | | Community |v0.3.0|
-| [Stripe](https://github.com/conduitio-labs/conduit-connector-stripe) |✅ | | Community |v0.3.0|
-| [Vitess](https://github.com/conduitio-labs/conduit-connector-vitess) |WIP|WIP| Community |WIP|
-| [Zendesk](https://github.com/conduitio-labs/conduit-connector-zendesk) |✅ |✅| Community |v0.3.0|
+A full list of connectors is hosted here: <https://conduit.io/docs/connectors/connector-list/>.

pkg/connector/service.go

@@ -29,6 +29,11 @@ import (
 
 var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`)
 
+const (
+	IDLengthLimit   = 256
+	NameLengthLimit = 256
+)
+
 // Service manages connectors.
 type Service struct {
 	logger log.CtxLogger
@@ -291,7 +296,7 @@ func (s *Service) validateConnector(cfg Config, id string) error {
 	if cfg.Name == "" {
 		multierr = multierror.Append(multierr, ErrNameMissing)
 	}
-	if len(cfg.Name) > 64 {
+	if len(cfg.Name) > NameLengthLimit {
 		multierr = multierror.Append(multierr, ErrNameOverLimit)
 	}
 	if id == "" {
@@ -301,7 +306,7 @@ func (s *Service) validateConnector(cfg Config, id string) error {
 	if !matched {
 		multierr = multierror.Append(multierr, ErrInvalidCharacters)
 	}
-	if len(id) > 64 {
+	if len(id) > IDLengthLimit {
 		multierr = multierror.Append(multierr, ErrIDOverLimit)
 	}
 

pkg/connector/service_test.go

@@ -17,6 +17,7 @@ package connector
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -308,16 +309,16 @@ func TestService_Create_ValidateSuccess(t *testing.T) {
 		data   Config
 	}{{
 		name:   "valid config name",
-		connID: uuid.NewString(),
+		connID: strings.Repeat("a", 128),
 		data: Config{
-			Name:     "Name#@-/_0%$",
+			Name:     strings.Repeat("a", 128),
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
 		name:   "valid connector ID",
 		connID: "Aa0-_.",
 		data: Config{
-			Name:     "test-connector",
+			Name:     "Name#@-/_0%$",
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}}
@@ -361,16 +362,16 @@ func TestService_Create_ValidateError(t *testing.T) {
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
-		name:    "connector name over 64 characters",
+		name:    "connector name over 256 characters",
 		connID:  uuid.NewString(),
 		errType: ErrNameOverLimit,
 		data: Config{
-			Name:     "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+			Name:     strings.Repeat("a", 257),
 			Settings: map[string]string{"foo": "bar"},
 		},
 	}, {
-		name:    "connector ID over 64 characters",
-		connID:  "aaaaaaaaa1bbbbbbbbb2ccccccccc3ddddddddd4eeeeeeeee5fffffffff6ggggg",
+		name:    "connector ID over 256 characters",
+		connID:  strings.Repeat("a", 257),
 		errType: ErrIDOverLimit,
 		data: Config{
 			Name:     "test-connector",

pkg/pipeline/service.go

@@ -29,6 +29,12 @@ import (
 
 var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`)
 
+const (
+	IDLengthLimit          = 128
+	NameLengthLimit        = 128
+	DescriptionLengthLimit = 8192
+)
+
 type FailureEvent struct {
 	// ID is the ID of the pipeline which failed.
 	ID    string
@@ -338,10 +344,10 @@ func (s *Service) validatePipeline(cfg Config, id string) error {
 	if s.instanceNames[cfg.Name] {
 		multierr = multierror.Append(multierr, ErrNameAlreadyExists)
 	}
-	if len(cfg.Name) > 64 {
+	if len(cfg.Name) > NameLengthLimit {
 		multierr = multierror.Append(multierr, ErrNameOverLimit)
 	}
-	if len(cfg.Description) > 8192 {
+	if len(cfg.Description) > DescriptionLengthLimit {
 		multierr = multierror.Append(multierr, ErrDescriptionOverLimit)
 	}
 	if id == "" {
@@ -351,7 +357,7 @@ func (s *Service) validatePipeline(cfg Config, id string) error {
 	if !matched {
 		multierr = multierror.Append(multierr, ErrInvalidCharacters)
 	}
-	if len(id) > 64 {
+	if len(id) > IDLengthLimit {
 		multierr = multierror.Append(multierr, ErrIDOverLimit)
 	}