diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 023eabca9a5..97d8099cf93 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,51 +1,82 @@ [bumpversion] -current_version = 0.50.23 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? serialize = {major}.{minor}.{patch} -[bumpversion:file:.bumpversion.cfg] - [bumpversion:file:.env] +search = VERSION=dev +replace = VERSION={new_version} [bumpversion:file:airbyte-container-orchestrator/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:airbyte-proxy/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:airbyte-server/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} -[bumpversion:file:airbyte-webapp/package.json] -search = "version": "{current_version}" -replace = "version": "{new_version}" +[bumpversion:file:airbyte-connector-builder-server/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:charts/airbyte-bootloader/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-cron/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-metrics/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-pod-sweeper/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-server/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-temporal/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-webapp/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-worker/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml] - -[bumpversion:file:charts/airbyte/README.md] - -[bumpversion:file:airbyte-connector-builder-server/Dockerfile] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-keycloak/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-keycloak-setup/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-api-server/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} + +[bumpversion:file:charts/airbyte/README.md] +search = ![appVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +replace = ![appVersion: {new_version}](https://img.shields.io/badge/AppVersion-{new_version}-informational?style=flat-square) diff --git a/.env b/.env index 2169a04ebe1..980c57ad3b9 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.50.23 +VERSION=dev # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/.gitignore b/.gitignore index 62657420c01..fa6dc52c190 100644 --- a/.gitignore +++ b/.gitignore @@ -63,9 +63,6 @@ crash.log resources/examples/airflow/logs/* !resources/examples/airflow/logs/.gitkeep -# Cloud Demo -!airbyte-webapp/src/packages/cloud/data - # Ignore docs folder, since we're using it to temporarily copy files into on CI /docs @@ -87,4 +84,4 @@ dd-java-agent.jar # Ignore airbyte.yml since this file contains user-provided information that is sensitive and should not be committed # See airbyte.sample.yml for an example of the expected file structure. -airbyte.yml +/configs/airbyte.yml diff --git a/airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java b/airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java index 52890686210..73ab2a94b5b 100644 --- a/airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java +++ b/airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java @@ -57,6 +57,7 @@ public class SegmentTrackingClient implements TrackingClient { private static final String AIRBYTE_TRACKED_AT = "tracked_at"; protected static final String UNKNOWN = "unknown"; protected static final String AIRBYTE_DEPLOYMENT_ID = "deployment_id"; + protected static final String AIRBYTE_DEPLOYMENT_MODE = "deployment_mode"; // Analytics is threadsafe. private final Analytics analytics; @@ -138,6 +139,7 @@ public void track(@Nullable final UUID workspaceId, final String action, final M mapCopy.put(AIRBYTE_VERSION_KEY, trackingIdentity.getAirbyteVersion().serialize()); mapCopy.put(CUSTOMER_ID_KEY, trackingIdentity.getCustomerId()); mapCopy.put(AIRBYTE_DEPLOYMENT_ID, deployment.getDeploymentId()); + mapCopy.put(AIRBYTE_DEPLOYMENT_MODE, deployment.getDeploymentMode()); mapCopy.put(AIRBYTE_TRACKED_AT, Instant.now().toString()); if (!metadata.isEmpty()) { trackingIdentity.getEmail().ifPresent(email -> mapCopy.put("email", email)); diff --git a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java index 02716fe66fa..99abb85f411 100644 --- a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java +++ b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java @@ -6,6 +6,7 @@ import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_ANALYTIC_SOURCE_HEADER; import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_DEPLOYMENT_ID; +import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_DEPLOYMENT_MODE; import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_SOURCE; import static io.airbyte.analytics.SegmentTrackingClient.AIRBYTE_VERSION_KEY; import static io.airbyte.analytics.SegmentTrackingClient.UNKNOWN; @@ -119,7 +120,8 @@ void testTrack() { ImmutableMap.of(AIRBYTE_VERSION_KEY, AIRBYTE_VERSION.serialize(), "user_id", IDENTITY.getCustomerId(), AIRBYTE_SOURCE, UNKNOWN, - AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId()); + AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId(), + AIRBYTE_DEPLOYMENT_MODE, DEPLOYMENT.getDeploymentMode()); segmentTrackingClient.track(WORKSPACE_ID, JUMP); @@ -139,7 +141,8 @@ void testTrackWithMetadata() { "height", "80 meters", "user_id", IDENTITY.getCustomerId(), AIRBYTE_SOURCE, UNKNOWN, - AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId()); + AIRBYTE_DEPLOYMENT_ID, DEPLOYMENT.getDeploymentId(), + AIRBYTE_DEPLOYMENT_MODE, DEPLOYMENT.getDeploymentMode()); segmentTrackingClient.track(WORKSPACE_ID, JUMP, metadata); diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/ApiTrackingHelpers.kt similarity index 81% rename from airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt rename to airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/ApiTrackingHelpers.kt index f50c8c78124..9d33a1f4cf7 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/ApiTrackingHelpers.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/ApiTrackingHelpers.kt @@ -1,7 +1,7 @@ /* * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.api.server.helpers +package io.airbyte.api.server.apiTracking import io.airbyte.analytics.Deployment import io.airbyte.analytics.TrackingClientSingleton @@ -31,13 +31,19 @@ private val USER_ID = "user_id" private val ENDPOINT = "endpoint" private val OPERATION = "operation" private val STATUS_CODE = "status_code" -private val API_VERSION = "api_version" private val WORKSPACE = "workspace" fun setupTrackingClient( - airbyteVersion: AirbyteVersion?, - trackingStrategy: Configs.TrackingStrategy?, + airbyteVersion: AirbyteVersion, + deploymentMode: Configs.DeploymentMode, + trackingStrategy: Configs.TrackingStrategy, + workerEnvironment: Configs.WorkerEnvironment, ) { + log.debug("deployment mode: $deploymentMode") + log.debug("airbyte version: $airbyteVersion") + log.debug("tracking strategy: $trackingStrategy") + log.debug("worker environment: $workerEnvironment") + // fake a deployment UUID until we want to have the public api server talking to the database // directly val deploymentId = UUID.randomUUID() @@ -45,9 +51,9 @@ fun setupTrackingClient( TrackingClientSingleton.initializeWithoutDatabase( trackingStrategy, Deployment( - Configs.DeploymentMode.OSS, + deploymentMode, deploymentId, - Configs.WorkerEnvironment.KUBERNETES, + workerEnvironment, ), airbyteVersion, ) @@ -58,7 +64,6 @@ fun track( endpointPath: String?, httpOperation: String?, httpStatusCode: Int, - apiVersion: String?, workspaceId: Optional, ) { val payload = mutableMapOf( @@ -66,7 +71,6 @@ fun track( Pair(ENDPOINT, endpointPath), Pair(OPERATION, httpOperation), Pair(STATUS_CODE, httpStatusCode), - Pair(API_VERSION, apiVersion), ) if (workspaceId.isPresent) { payload[WORKSPACE] = workspaceId.get().toString() diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingClient.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingClient.kt new file mode 100644 index 00000000000..7a143885b03 --- /dev/null +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingClient.kt @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.api.server.apiTracking + +import io.airbyte.config.Configs +import io.airbyte.config.EnvConfigs +import jakarta.inject.Singleton + +/** + * Used to track usage of the airbyte-api-server. + */ +@Singleton +class TrackingClient { + val configs: Configs = EnvConfigs() + + init { + setupTrackingClient(configs.airbyteVersion, configs.deploymentMode, configs.trackingStrategy, configs.workerEnvironment) + } +} diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt similarity index 94% rename from airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt rename to airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt index 1bd47ce736a..a0d4d4a02aa 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/TrackingHelper.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/apiTracking/TrackingHelper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.api.server.helpers +package io.airbyte.api.server.apiTracking import io.micronaut.http.HttpStatus import org.zalando.problem.AbstractThrowableProblem @@ -16,7 +16,6 @@ import javax.ws.rs.core.Response * Todo: This should be a middleware through a micronaut annotation so that we do not need to add this wrapper functions around all our calls. */ object TrackingHelper { - private val API_VERSION = System.getenv("AIRBYTE_VERSION") ?: "unknown" private fun trackSuccess(endpointPath: String, httpOperation: String, userId: UUID, workspaceId: Optional) { val statusCode = Response.Status.OK.statusCode track( @@ -24,7 +23,6 @@ object TrackingHelper { endpointPath, httpOperation, statusCode, - API_VERSION, workspaceId, ) } @@ -62,7 +60,6 @@ object TrackingHelper { endpointPath, httpOperation, statusCode, - API_VERSION, Optional.empty(), ) } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt index accd5a70157..30fc63bd5c1 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt @@ -13,6 +13,7 @@ import io.airbyte.api.client.model.generated.AirbyteCatalog import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration import io.airbyte.api.client.model.generated.DestinationSyncMode import io.airbyte.api.client.model.generated.SourceDiscoverSchemaRead +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.CONNECTIONS_PATH import io.airbyte.api.server.constants.CONNECTIONS_WITH_ID_PATH import io.airbyte.api.server.constants.DELETE @@ -20,7 +21,6 @@ import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.POST import io.airbyte.api.server.constants.PUT import io.airbyte.api.server.helpers.AirbyteCatalogHelper -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.services.ConnectionService import io.airbyte.api.server.services.DestinationService diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt index 8d7dc183865..9daca8b3bf5 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt @@ -9,6 +9,7 @@ import io.airbyte.airbyte_api.generated.DestinationsApi import io.airbyte.airbyte_api.model.generated.DestinationCreateRequest import io.airbyte.airbyte_api.model.generated.DestinationPatchRequest import io.airbyte.airbyte_api.model.generated.DestinationPutRequest +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.DELETE import io.airbyte.api.server.constants.DESTINATIONS_PATH import io.airbyte.api.server.constants.DESTINATIONS_WITH_ID_PATH @@ -17,7 +18,6 @@ import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.PATCH import io.airbyte.api.server.constants.POST import io.airbyte.api.server.constants.PUT -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getIdFromName import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.helpers.removeDestinationType diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt index d34940cb50e..a059924db0a 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt @@ -11,13 +11,14 @@ import io.airbyte.airbyte_api.model.generated.JobStatusEnum import io.airbyte.airbyte_api.model.generated.JobTypeEnum import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByFieldEnum import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByMethodEnum +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.JOBS_PATH import io.airbyte.api.server.constants.JOBS_WITH_ID_PATH import io.airbyte.api.server.constants.POST import io.airbyte.api.server.filters.JobsFilter -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.problems.BadRequestProblem import io.airbyte.api.server.problems.UnprocessableEntityProblem @@ -27,6 +28,11 @@ import io.airbyte.api.server.services.UserService import io.micronaut.http.annotation.Controller import java.time.OffsetDateTime import java.util.UUID +import javax.ws.rs.DELETE +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.PathParam import javax.ws.rs.core.Response @Controller(JOBS_PATH) @@ -35,7 +41,13 @@ open class JobsController( private val userService: UserService, private val connectionService: ConnectionService, ) : JobsApi { - override fun cancelJob(jobId: Long, userInfo: String?): Response { + + @DELETE + @Path("/{jobId}") + override fun cancelJob( + @PathParam("jobId") jobId: Long, + @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, + ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) val jobResponse: Any? = TrackingHelper.callWithTracker( @@ -130,7 +142,12 @@ open class JobsController( } } - override fun getJob(jobId: Long, userInfo: String?): Response { + @GET + @Path("/{jobId}") + override fun getJob( + @PathParam("jobId") jobId: Long, + @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, + ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) val jobResponse: Any? = TrackingHelper.callWithTracker( diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt index 9ce3294b653..49b85c48e6f 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt @@ -10,6 +10,7 @@ import io.airbyte.airbyte_api.model.generated.InitiateOauthRequest import io.airbyte.airbyte_api.model.generated.SourceCreateRequest import io.airbyte.airbyte_api.model.generated.SourcePatchRequest import io.airbyte.airbyte_api.model.generated.SourcePutRequest +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.DELETE import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.PATCH @@ -18,7 +19,6 @@ import io.airbyte.api.server.constants.PUT import io.airbyte.api.server.constants.SOURCES_PATH import io.airbyte.api.server.constants.SOURCES_WITH_ID_PATH import io.airbyte.api.server.constants.SOURCE_TYPE -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getIdFromName import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.helpers.removeSourceTypeNode diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt index 681b2ca5ef1..897c8cf4ac7 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt @@ -13,9 +13,9 @@ import io.airbyte.airbyte_api.model.generated.StreamProperties import io.airbyte.api.client.model.generated.AirbyteStreamAndConfiguration import io.airbyte.api.client.model.generated.DestinationSyncMode import io.airbyte.api.client.model.generated.SyncMode +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.STREAMS_PATH -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.problems.UnexpectedProblem import io.airbyte.api.server.services.DestinationService diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt index 867766aeedd..fc2a94bf6c9 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt @@ -9,12 +9,12 @@ import io.airbyte.airbyte_api.model.generated.WorkspaceCreateRequest import io.airbyte.airbyte_api.model.generated.WorkspaceOAuthCredentialsRequest import io.airbyte.airbyte_api.model.generated.WorkspaceResponse import io.airbyte.airbyte_api.model.generated.WorkspaceUpdateRequest +import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.DELETE import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.POST import io.airbyte.api.server.constants.WORKSPACES_PATH import io.airbyte.api.server.constants.WORKSPACES_WITH_ID_PATH -import io.airbyte.api.server.helpers.TrackingHelper import io.airbyte.api.server.helpers.getLocalUserInfoIfNull import io.airbyte.api.server.services.UserService import io.airbyte.api.server.services.WorkspaceService diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt index 15335e8ee30..4d1244f3606 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt @@ -135,10 +135,14 @@ object AirbyteCatalogHelper { // Ensure minutes value is not `*` Integer.valueOf(cronStrings[1]) } + } catch (e: NumberFormatException) { + log.debug("Invalid cron expression: " + connectionSchedule.cronExpression) + log.debug("NumberFormatException: $e") + throw ConnectionConfigurationProblem.invalidCronExpressionUnderOneHour(connectionSchedule.cronExpression) } catch (e: IllegalArgumentException) { log.debug("Invalid cron expression: " + connectionSchedule.cronExpression) log.debug("IllegalArgumentException: $e") - throw ConnectionConfigurationProblem.invalidCronExpression(connectionSchedule.cronExpression) + throw ConnectionConfigurationProblem.invalidCronExpression(connectionSchedule.cronExpression, e.message) } } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt index 284822c62a3..3c35c0f2438 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt @@ -85,13 +85,20 @@ class ConnectionConfigurationProblem private constructor(message: String) : Abst ) } - fun invalidCronExpression(cronExpression: String): ConnectionConfigurationProblem { + fun invalidCronExpressionUnderOneHour(cronExpression: String): ConnectionConfigurationProblem { return ConnectionConfigurationProblem( "The cron expression " + cronExpression + " is not valid or is less than the one hour minimum. The seconds and minutes values cannot be `*`.", ) } + fun invalidCronExpression(cronExpression: String, message: String?): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "The cron expression $cronExpression is not valid. Error: $message" + + ". Please check the cron expression format at https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html", + ) + } + fun missingCronExpression(): ConnectionConfigurationProblem { return ConnectionConfigurationProblem("Missing cron expression in the schedule.") } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt index c6cfcc4a9ba..ba1e616fcb4 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt @@ -84,6 +84,12 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi } catch (e: HttpClientResponseException) { log.error("Config api response error for job sync: ", e) e.response as HttpResponse + } catch (e: ReadTimeoutException) { + log.error("Config api response error for job sync: ", e) + throw UnexpectedProblem( + HttpStatus.REQUEST_TIMEOUT, + "Request timed out. Please check the latest job status to determine whether the sync started.", + ) } ConfigClientErrorHandler.handleError(response, connectionId.toString()) log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) @@ -168,9 +174,9 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi .createdAtEnd(jobsFilter.createdAtEnd) .updatedAtStart(jobsFilter.updatedAtStart) .updatedAtEnd(jobsFilter.updatedAtEnd) - .orderByField(JobListRequestBody.OrderByFieldEnum.fromValue(orderByField.name)) + .orderByField(JobListRequestBody.OrderByFieldEnum.valueOf(orderByField.name)) .orderByMethod( - JobListRequestBody.OrderByMethodEnum.fromValue(orderByMethod.name), + JobListRequestBody.OrderByMethodEnum.valueOf(orderByMethod.name), ) val response = try { @@ -215,8 +221,8 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi .createdAtEnd(jobsFilter.createdAtEnd) .updatedAtStart(jobsFilter.updatedAtStart) .updatedAtEnd(jobsFilter.updatedAtEnd) - .orderByField(OrderByFieldEnum.fromValue(orderByField.name)) - .orderByMethod(OrderByMethodEnum.fromValue(orderByMethod.name)) + .orderByField(OrderByFieldEnum.valueOf(orderByField.name)) + .orderByMethod(OrderByMethodEnum.valueOf(orderByMethod.name)) val response = try { configApiClient.getJobListForWorkspaces(requestBody, userInfo) diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index 8bee489ce57..cf82c373508 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -18,7 +18,7 @@ micronaut: idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} http: client: - read-timeout: 20s + read-timeout: ${READ_TIMEOUT:150s} max-content-length: 52428800 # 50MB airbyte: diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index 2962831ec9c..1950d921c53 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -31,6 +31,8 @@ import org.slf4j.LoggerFactory; /** + * DEPRECATED. USE {@link io.airbyte.api.client2.AirbyteApiClient2}. + *

* This class is meant to consolidate all our API endpoints into a fluent-ish client. Currently, all * open API generators create a separate class per API "root-route". For example, if our API has two * routes "/v1/First/get" and "/v1/Second/get", OpenAPI generates (essentially) the following files: @@ -41,7 +43,10 @@ * ApiClient()).get(), which can get cumbersome if we're interacting with many pieces of the API. *

* This is currently manually maintained. We could look into autogenerating it if needed. + * + * @deprecated Replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ +@Deprecated public class AirbyteApiClient { private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteApiClient.class); @@ -176,16 +181,27 @@ public OrganizationApi getOrganizationApi() { } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Default to 4 retries with a randomised 1 - 10 seconds interval between the first two retries and * an 10-minute wait for the last retry. *

* Exceptions will be swallowed. + * + * @param call method to execute + * @param desc short readable explanation of why this method is executed + * @param type of return type + * @return value returned by method + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated public static T retryWithJitter(final Callable call, final String desc) { return retryWithJitter(call, desc, DEFAULT_RETRY_INTERVAL_SECS, DEFAULT_FINAL_INTERVAL_SECS, DEFAULT_MAX_RETRIES); } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Provides a simple retry wrapper for api calls. This retry behaviour is slightly different from * generally available retries libraries - the last retry is able to wait an interval inconsistent * with regular intervals/exponential backoff. @@ -200,7 +216,9 @@ public static T retryWithJitter(final Callable call, final String desc) { * @param desc short readable explanation of why this method is executed * @param jitterMaxIntervalSecs upper limit of the randomised retry interval. Minimum value is 1. * @param finalIntervalSecs retry interval before the last retry. + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated @VisibleForTesting // This is okay since we are logging the stack trace, which PMD is not detecting. @SuppressWarnings("PMD.PreserveStackTrace") @@ -219,14 +237,26 @@ public static T retryWithJitter(final Callable call, } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Default to 4 retries with a randomised 1 - 10 seconds interval between the first two retries and * an 10-minute wait for the last retry. + * + * @param call method to execute + * @param desc description of what is happening + * @param type of return type + * @return value returned by method + * @throws Exception exception while jittering + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated public static T retryWithJitterThrows(final Callable call, final String desc) throws Exception { return retryWithJitterThrows(call, desc, DEFAULT_RETRY_INTERVAL_SECS, DEFAULT_FINAL_INTERVAL_SECS, DEFAULT_MAX_RETRIES); } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Provides a simple retry wrapper for api calls. This retry behaviour is slightly different from * generally available retries libraries - the last retry is able to wait an interval inconsistent * with regular intervals/exponential backoff. @@ -239,8 +269,10 @@ public static T retryWithJitterThrows(final Callable call, final String d * @param desc short readable explanation of why this method is executed * @param jitterMaxIntervalSecs upper limit of the randomised retry interval. Minimum value is 1. * @param finalIntervalSecs retry interval before the last retry. + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ @VisibleForTesting + @Deprecated // This is okay since we are logging the stack trace, which PMD is not detecting. @SuppressWarnings("PMD.PreserveStackTrace") public static T retryWithJitterThrows(final Callable call, diff --git a/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt b/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt index c9a396c5fba..d64b1ce8e1d 100644 --- a/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt +++ b/airbyte-api/src/main/kotlin/AirbyteApiClient2.kt @@ -52,39 +52,20 @@ class AirbyteApiClient2 httpClient: OkHttpClient = OkHttpClient(), ) { - val connectionApi: ConnectionApi - val connectorBuilderProjectApi: ConnectorBuilderProjectApi - val destinationDefinitionApi: DestinationDefinitionApi - val destinationApi: DestinationApi - val destinationSpecificationApi: DestinationDefinitionSpecificationApi - val jobsApi: JobsApi - val jobRetryStatesApi: JobRetryStatesApi - val operationApi: OperationApi - val sourceDefinitionApi: SourceDefinitionApi - val sourceApi: SourceApi - val sourceDefinitionSpecificationApi: SourceDefinitionSpecificationApi - val workspaceApi: WorkspaceApi - val healthApi: HealthApi - val attemptApi: AttemptApi - val stateApi: StateApi - val streamStatusesApi: StreamStatusesApi - - init { - connectionApi = ConnectionApi(basePath = basePath, client = httpClient, policy = policy) - connectorBuilderProjectApi = ConnectorBuilderProjectApi(basePath = basePath, client = httpClient, policy = policy) - destinationDefinitionApi = DestinationDefinitionApi(basePath = basePath, client = httpClient, policy = policy) - destinationApi = DestinationApi(basePath = basePath, client = httpClient, policy = policy) - destinationSpecificationApi = DestinationDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy) - jobsApi = JobsApi(basePath = basePath, client = httpClient, policy = policy) - jobRetryStatesApi = JobRetryStatesApi(basePath = basePath, client = httpClient, policy = policy) - operationApi = OperationApi(basePath = basePath, client = httpClient, policy = policy) - sourceDefinitionApi = SourceDefinitionApi(basePath = basePath, client = httpClient, policy = policy) - sourceApi = SourceApi(basePath = basePath, client = httpClient, policy = policy) - sourceDefinitionSpecificationApi = SourceDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy) - workspaceApi = WorkspaceApi(basePath = basePath, client = httpClient, policy = policy) - healthApi = HealthApi(basePath = basePath, client = httpClient, policy = policy) - attemptApi = AttemptApi(basePath = basePath, client = httpClient, policy = policy) - stateApi = StateApi(basePath = basePath, client = httpClient, policy = policy) - streamStatusesApi = StreamStatusesApi(basePath = basePath, client = httpClient, policy = policy) - } + val connectionApi = ConnectionApi(basePath = basePath, client = httpClient, policy = policy) + val connectorBuilderProjectApi = ConnectorBuilderProjectApi(basePath = basePath, client = httpClient, policy = policy) + val destinationDefinitionApi = DestinationDefinitionApi(basePath = basePath, client = httpClient, policy = policy) + val destinationApi = DestinationApi(basePath = basePath, client = httpClient, policy = policy) + val destinationSpecificationApi = DestinationDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy) + val jobsApi = JobsApi(basePath = basePath, client = httpClient, policy = policy) + val jobRetryStatesApi = JobRetryStatesApi(basePath = basePath, client = httpClient, policy = policy) + val operationApi = OperationApi(basePath = basePath, client = httpClient, policy = policy) + val sourceDefinitionApi = SourceDefinitionApi(basePath = basePath, client = httpClient, policy = policy) + val sourceApi = SourceApi(basePath = basePath, client = httpClient, policy = policy) + val sourceDefinitionSpecificationApi = SourceDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy) + val workspaceApi = WorkspaceApi(basePath = basePath, client = httpClient, policy = policy) + val healthApi = HealthApi(basePath = basePath, client = httpClient, policy = policy) + val attemptApi = AttemptApi(basePath = basePath, client = httpClient, policy = policy) + val stateApi = StateApi(basePath = basePath, client = httpClient, policy = policy) + val streamStatusesApi = StreamStatusesApi(basePath = basePath, client = httpClient, policy = policy) } diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml index 1dc9d22f369..64aacc4e132 100644 --- a/airbyte-api/src/main/openapi/cloud-config.yaml +++ b/airbyte-api/src/main/openapi/cloud-config.yaml @@ -239,9 +239,9 @@ paths: application/json: schema: $ref: "#/components/schemas/CloudWorkspaceReadList" - /v1/cloud_workspaces/list_workspaces_by_most_recently_running_jobs: + /v1/cloud_workspaces/list_active_workspaces_by_most_recently_running_jobs: post: - operationId: listWorkspacesByMostRecentlyRunningJobs + operationId: listActiveWorkspacesByMostRecentlyRunningJobs requestBody: content: application/json: @@ -1006,6 +1006,8 @@ components: enum: # - auth0 - google_identity_platform + - airbyte + - keycloak # WORKSPACE WorkspaceUserRead: type: object @@ -1411,6 +1413,13 @@ components: - generally_available - custom + ActorSupportLevel: + type: string + enum: + - community + - certified + - none + ConnectionProto: type: object required: @@ -1423,12 +1432,16 @@ components: - sourceConnectionName - sourceIcon - sourceReleaseStage + - sourceSupportLevel + - sourceCustom - destinationId - destinationDefinitionId - destinationDefinitionName - destinationConnectionName - destinationIcon - destinationReleaseStage + - destinationSupportLevel + - destinationCustom properties: connectionId: type: string @@ -1460,6 +1473,13 @@ components: sourceIcon: description: Icon of the source actor. type: string + sourceCustom: + description: True if the source is custom + type: boolean + default: false + sourceSupportLevel: + description: Support Level of source actor. + $ref: "#/components/schemas/ActorSupportLevel" sourceReleaseStage: description: Release stage of source actor. $ref: "#/components/schemas/ActorReleaseStage" @@ -1478,6 +1498,13 @@ components: destinationIcon: description: Icon of the destination actor. type: string + destinationCustom: + description: True if the destination is custom + type: boolean + default: false + destinationSupportLevel: + description: Support Level of destination actor. + $ref: "#/components/schemas/ActorSupportLevel" destinationReleaseStage: description: Release stage of destination actor. $ref: "#/components/schemas/ActorReleaseStage" diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 09c3b408a82..1d790f81065 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -137,7 +137,7 @@ paths: post: tags: - workspace - summary: List all workspaces registered in the current Airbyte deployment, paginated. + summary: List all workspaces registered in the current Airbyte deployment. This function also supports searching by keyword and pagination. operationId: listAllWorkspacesPaginated requestBody: content: @@ -156,7 +156,7 @@ paths: post: tags: - workspace - summary: List all workspaces registered in the current Airbyte deployment, paginated + summary: List workspaces by given workspace IDs registered in the current Airbyte deployment. This function also supports pagination. operationId: listWorkspacesPaginated requestBody: content: @@ -173,7 +173,7 @@ paths: $ref: "#/components/schemas/WorkspaceReadList" /v1/workspaces/list_by_organization_id: post: - summary: List workspaces under the given org id + summary: List workspaces under the given org id. This function also supports searching by keyword and pagination. tags: - workspace operationId: listWorkspacesInOrganization @@ -191,7 +191,26 @@ paths: $ref: "#/components/schemas/WorkspaceReadList" "404": $ref: "#/components/responses/NotFoundResponse" - + /v1/workspaces/list_by_user_id: + post: + summary: List workspaces by a given user id. The function also supports searching by keyword and pagination. + tags: + - workspace + operationId: listWorkspacesByUser + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListWorkspacesByUserRequestBody" + responses: + "200": + description: Successfully retrieved workspaces by given user id. + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" /v1/workspaces/get: post: tags: @@ -3105,6 +3124,22 @@ paths: "404": $ref: "#/components/responses/NotFoundResponse" + /v1/users/list_instance_admin: + post: + summary: List all users with instance admin permissions. Only instance admin has permission to call this. + tags: + - user + operationId: listInstanceAdminUsers + responses: + "200": + description: List all instance admin users. + content: + application/json: + schema: + $ref: "#/components/schemas/UserWithPermissionInfoReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + # PERMISSIONS /v1/permissions/create: post: @@ -3313,6 +3348,37 @@ paths: schema: type: string format: binary + /v1/attempt/create_new_attempt_number: + post: + tags: + - attempt + - internal + summary: For worker to create a new attempt number. + operationId: createNewAttemptNumber + requestBody: + content: + application/json: + schema: + type: object + required: + - jobId + properties: + jobId: + $ref: "#/components/schemas/JobId" + required: true + responses: + "200": + description: Successful Operation + content: + application/json: + schema: + title: CreateNewAttemptNumberResponse + type: object + required: + - attemptNumber + properties: + attemptNumber: + $ref: "#/components/schemas/AttemptNumber" /v1/attempt/set_workflow_in_attempt: post: tags: @@ -3515,6 +3581,27 @@ paths: $ref: "#/components/schemas/InstanceConfigurationResponse" "401": description: Fetching instance configuration failed. + /v1/instance_configuration/setup: + post: + summary: Setup an instance with user and organization information. + tags: + - instance_configuration + operationId: setupInstanceConfiguration + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceConfigurationSetupRequestBody" + responses: + "200": + description: Successfully setup instance. + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceConfigurationResponse" + "401": + description: Instance setup failed. + /v1/jobs/retry_states/create_or_update: post: summary: Creates or updates a retry state for a job. @@ -3709,6 +3796,10 @@ components: $ref: "#/components/schemas/NotificationItem" sendOnConnectionUpdateActionRequired: $ref: "#/components/schemas/NotificationItem" + sendOnBreakingChangeWarning: + $ref: "#/components/schemas/NotificationItem" + sendOnBreakingChangeSyncsDisabled: + $ref: "#/components/schemas/NotificationItem" Notification: type: object @@ -3755,6 +3846,8 @@ components: - sync_disabled_warning - connection_update - connection_update_action_required + - breaking_change_warning + - breaking_change_syncs_disabled NotificationRead: type: object required: @@ -3796,6 +3889,7 @@ components: type: object required: - workspaceIds + - pagination properties: workspaceIds: type: array @@ -4180,6 +4274,12 @@ components: protocolVersion: description: The Airbyte Protocol version supported by the connector type: string + custom: + description: Whether the connector is custom or not + type: boolean + default: false + supportLevel: + $ref: "#/components/schemas/SupportLevel" releaseStage: $ref: "#/components/schemas/ReleaseStage" releaseDate: @@ -4657,6 +4757,12 @@ components: protocolVersion: description: The Airbyte Protocol version supported by the connector type: string + custom: + description: Whether the connector is custom or not + type: boolean + default: false + supportLevel: + $ref: "#/components/schemas/SupportLevel" releaseStage: $ref: "#/components/schemas/ReleaseStage" releaseDate: @@ -4902,15 +5008,23 @@ components: required: - dockerRepository - dockerImageTag + - supportsDbt + - normalizationConfig - supportState - - isActorDefaultVersion + - isOverrideApplied properties: dockerRepository: type: string dockerImageTag: type: string - isActorDefaultVersion: + supportsDbt: + type: boolean + normalizationConfig: + $ref: "#/components/schemas/NormalizationDestinationDefinitionConfig" + isOverrideApplied: type: boolean + supportLevel: + $ref: "#/components/schemas/SupportLevel" supportState: $ref: "#/components/schemas/SupportState" breakingChanges: @@ -4960,6 +5074,12 @@ components: - beta - generally_available - custom + SupportLevel: + type: string + enum: + - community + - certified + - none # CONNECTION ConnectionId: type: string @@ -6678,7 +6798,7 @@ components: actorType: $ref: "#/components/schemas/ActorType" NormalizationDestinationDefinitionConfig: - description: describes a normalization config for destination definition + description: describes a normalization config for destination definition version type: object required: - supported @@ -7492,6 +7612,10 @@ components: required: - edition - webappUrl + - initialSetupComplete + - defaultUserId + - defaultOrganizationId + - defaultWorkspaceId properties: edition: type: string @@ -7508,6 +7632,39 @@ components: $ref: "#/components/schemas/AuthConfiguration" webappUrl: type: string + initialSetupComplete: + type: boolean + defaultUserId: + type: string + format: uuid + defaultOrganizationId: + type: string + format: uuid + defaultWorkspaceId: + type: string + format: uuid + InstanceConfigurationSetupRequestBody: + type: object + required: + - email + - anonymousDataCollection + - initialSetupComplete + - displaySetupWizard + properties: + email: + type: string + anonymousDataCollection: + type: boolean + initialSetupComplete: + type: boolean + displaySetupWizard: + type: boolean + userName: + description: Optional name of the user to create. Defaults to 'Default User' if not specified. + type: string + organizationName: + description: Optional name of the organization to create. Defaults to 'Default Organization' if not specified. + type: string StreamStatusRead: type: object required: @@ -7652,6 +7809,18 @@ components: $ref: "#/components/schemas/Pagination" keyword: type: string + ListWorkspacesByUserRequestBody: + type: object + required: + - userId + properties: + userId: + type: string + format: uuid + pagination: + $ref: "#/components/schemas/Pagination" + keyword: + type: string OrganizationRead: type: object required: @@ -7734,7 +7903,29 @@ components: $ref: "#/components/schemas/PermissionType" organizationId: $ref: "#/components/schemas/OrganizationId" - + UserWithPermissionInfoReadList: + type: object + required: + - users + properties: + users: + type: array + items: + $ref: "#/components/schemas/UserWithPermissionInfoRead" + UserWithPermissionInfoRead: + type: object + properties: + name: + description: Caption name for the user + type: string + userId: + $ref: "#/components/schemas/UserId" + email: + type: string + format: email + permissionId: + type: string + format: uuid InvalidInputProperty: type: object required: diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java index 9fa87aa361f..6ebfb878d0b 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java @@ -13,6 +13,7 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.init.PostLoadExecutor; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; import io.airbyte.db.init.DatabaseInitializationException; import io.airbyte.db.init.DatabaseInitializer; import io.airbyte.db.instance.DatabaseMigrator; @@ -188,7 +189,9 @@ private void createWorkspaceIfNoneExists(final ConfigRepository configRepository .withInitialSetupComplete(false) .withDisplaySetupWizard(true) .withTombstone(false) - .withDefaultGeography(Geography.AUTO); + .withDefaultGeography(Geography.AUTO) + // attach this new workspace to the Default Organization which should always exist at this point. + .withOrganizationId(OrganizationPersistence.DEFAULT_ORGANIZATION_ID); // NOTE: it's safe to use the NoSecrets version since we know that the user hasn't supplied any // secrets yet. configRepository.writeStandardWorkspaceNoSecrets(workspace); diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java index 6dcea132e30..bbe9f3c8aed 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java @@ -6,6 +6,9 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; import io.airbyte.db.Database; import io.airbyte.db.check.impl.JobsDatabaseAvailabilityCheck; import io.airbyte.db.factory.DatabaseCheckFactory; @@ -16,7 +19,9 @@ import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; import io.micronaut.flyway.FlywayConfigurationProperties; @@ -107,6 +112,11 @@ public JobPersistence jobPersistence(@Named("jobsDatabase") final Database jobDa return new DefaultJobPersistence(jobDatabase); } + @Singleton + public MetadataPersistence metadataPersistence(@Named("jobsDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + @SuppressWarnings("LineLength") @Singleton @Named("configsDatabaseInitializer") @@ -147,4 +157,19 @@ public DatabaseMigrator jobsDatabaseMigrator(@Named("jobsDatabase") final Databa return new JobsDatabaseMigrator(jobsDatabase, jobsFlyway); } + @Singleton + public UserPersistence userPersistence(@Named("configDatabase") final Database configDatabase) { + return new UserPersistence(configDatabase); + } + + @Singleton + public OrganizationPersistence organizationPersistence(@Named("configDatabase") final Database configDatabase) { + return new OrganizationPersistence(configDatabase); + } + + @Singleton + public WorkspacePersistence workspacePersistence(@Named("configDatabase") final Database configDatabase) { + return new WorkspacePersistence(configDatabase); + } + } diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index ff56104a407..d4a0a5666da 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -15,11 +15,14 @@ import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.commons.version.Version; +import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.init.ApplyDefinitionsHelper; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.init.DeclarativeSourceUpdater; import io.airbyte.config.init.PostLoadExecutor; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.config.specs.LocalDefinitionsProvider; import io.airbyte.db.factory.DSLContextFactory; @@ -75,7 +78,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.20.001"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.24.001"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; @@ -132,7 +135,11 @@ void testBootloaderAppBlankDb() throws Exception { val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); - val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient); + val overrideProvider = new NoOpDefinitionVersionOverrideProvider(); + val actorDefinitionVersionHelper = new ActorDefinitionVersionHelper(configRepository, overrideProvider, featureFlagClient); + val supportStateUpdater = new SupportStateUpdater(configRepository, actorDefinitionVersionHelper, DeploymentMode.OSS, featureFlagClient); + val applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); final CdkVersionProvider cdkVersionProvider = mock(CdkVersionProvider.class); when(cdkVersionProvider.getCdkVersion()).thenReturn(CDK_VERSION); val declarativeSourceUpdater = new DeclarativeSourceUpdater(configRepository, cdkVersionProvider); @@ -185,8 +192,12 @@ void testRequiredVersionUpgradePredicate() throws Exception { jobsDatabaseInitializationTimeoutMs, MoreResources.readResource(DatabaseConstants.JOBS_INITIAL_SCHEMA_PATH)); val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); + val overrideProvider = new NoOpDefinitionVersionOverrideProvider(); + val actorDefinitionVersionHelper = new ActorDefinitionVersionHelper(configRepository, overrideProvider, featureFlagClient); + val supportStateUpdater = new SupportStateUpdater(configRepository, actorDefinitionVersionHelper, DeploymentMode.OSS, featureFlagClient); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); - val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient); + val applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); final CdkVersionProvider cdkVersionProvider = mock(CdkVersionProvider.class); when(cdkVersionProvider.getCdkVersion()).thenReturn(CDK_VERSION); val declarativeSourceUpdater = new DeclarativeSourceUpdater(configRepository, cdkVersionProvider); diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java new file mode 100644 index 00000000000..867d77ce68f --- /dev/null +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.bootloader; + +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.ActorType; +import io.airbyte.config.persistence.version_overrides.DefinitionVersionOverrideProvider; +import java.util.Optional; +import java.util.UUID; +import org.jetbrains.annotations.Nullable; + +/** + * Implementation of {@link DefinitionVersionOverrideProvider} that does not override any versions. + * Used for testing. + */ +class NoOpDefinitionVersionOverrideProvider implements DefinitionVersionOverrideProvider { + + @Override + public Optional getOverride(final ActorType actorType, + final UUID actorDefinitionId, + final UUID workspaceId, + @Nullable final UUID actorId, + final ActorDefinitionVersion defaultVersion) { + return Optional.empty(); + } + +} diff --git a/airbyte-commons-license/build.gradle b/airbyte-commons-license/build.gradle index 7cf60b23f9b..cd78c944d64 100644 --- a/airbyte-commons-license/build.gradle +++ b/airbyte-commons-license/build.gradle @@ -14,6 +14,8 @@ dependencies { annotationProcessor libs.lombok implementation project(':airbyte-commons') + implementation project(':airbyte-commons-micronaut') + implementation project(':airbyte-config:config-models') testAnnotationProcessor platform(libs.micronaut.bom) testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor diff --git a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java index 7f195a7c5fb..091343599b4 100644 --- a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java +++ b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java @@ -18,7 +18,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) +@Target({ElementType.TYPE, ElementType.METHOD}) @Inherited @Requires(condition = AirbyteProEnabledCondition.class) public @interface RequiresAirbyteProEnabled { diff --git a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java index f9cb6932ebc..6572a7b1ec8 100644 --- a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java +++ b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java @@ -4,6 +4,7 @@ package io.airbyte.commons.license.condition; +import io.airbyte.config.Configs.AirbyteEdition; import io.micronaut.context.condition.Condition; import io.micronaut.context.condition.ConditionContext; import lombok.extern.slf4j.Slf4j; @@ -18,10 +19,8 @@ public class AirbyteProEnabledCondition implements Condition { @Override public boolean matches(ConditionContext context) { - log.warn("inside the pro enabled condition!"); - final var edition = context.getProperty("airbyte.edition", String.class).orElse("community"); - log.warn("got edition: " + edition); - return "pro".equals(edition); + final AirbyteEdition edition = context.getBean(AirbyteEdition.class); + return edition.equals(AirbyteEdition.PRO); } } diff --git a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java index 80f9abe3369..7c686a3535f 100644 --- a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java +++ b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java @@ -5,6 +5,7 @@ package io.airbyte.micronaut.config; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.Configs.AirbyteEdition; import io.airbyte.config.Configs.DeploymentMode; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; @@ -33,4 +34,12 @@ public DeploymentMode deploymentMode(@Value("${airbyte.deployment-mode}") final return convertToEnum(deploymentMode, DeploymentMode::valueOf, DeploymentMode.OSS); } + /** + * Fetch the configured edition of the Airbyte instance. Defaults to COMMUNITY. + */ + @Singleton + public AirbyteEdition airbyteEdition(@Value("${airbyte.edition:COMMUNITY}") final String airbyteEdition) { + return convertToEnum(airbyteEdition.toUpperCase(), AirbyteEdition::valueOf, AirbyteEdition.COMMUNITY); + } + } diff --git a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java index 3967d0bd96e..1901182c841 100644 --- a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java +++ b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java @@ -13,9 +13,9 @@ import io.micronaut.core.io.file.DefaultFileSystemResourceLoader; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; /** @@ -26,7 +26,7 @@ @Slf4j public class AirbytePropertySourceLoader implements PropertySourceLoader { - static final String AIRBYTE_YML_PATH = "/app/airbyte.yml"; + static final String AIRBYTE_YML_PATH = "/app/configs/airbyte.yml"; static final String AIRBYTE_KEY = "airbyte"; final YamlPropertySourceLoader yamlLoader = new YamlPropertySourceLoader(); @@ -50,8 +50,10 @@ public Optional load(final String resourceName, final ResourceLo final var yaml = read(resourceName, stream); // Prefix all configuration from that file with airbyte. - final var prefixedProps = yaml.entrySet().stream().collect(Collectors.toMap(entry -> AIRBYTE_KEY + "." + entry.getKey(), Map.Entry::getValue)); - log.info("Loading properties as: {}", prefixedProps); + final var prefixedProps = new HashMap(); + for (var entry : yaml.entrySet()) { + prefixedProps.put(AIRBYTE_KEY + "." + entry.getKey(), entry.getValue()); + } return Optional.of(PropertySource.of(prefixedProps)); } catch (final IOException e) { throw new ConfigurationException("Could not load airbyte.yml configuration file.", e); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java index cb784d74f27..1b465164f62 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java @@ -22,6 +22,7 @@ import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig; import io.airbyte.api.model.generated.ReleaseStage; import io.airbyte.api.model.generated.ResourceRequirements; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.SupportState; import io.airbyte.commons.converters.StateConverter; import io.airbyte.commons.enums.Enums; @@ -234,6 +235,13 @@ public static ReleaseStage toApiReleaseStage(final io.airbyte.config.ReleaseStag return ReleaseStage.fromValue(releaseStage.value()); } + public static SupportLevel toApiSupportLevel(final io.airbyte.config.SupportLevel supportLevel) { + if (supportLevel == null) { + return SupportLevel.NONE; + } + return SupportLevel.fromValue(supportLevel.value()); + } + public static SupportState toApiSupportState(final io.airbyte.config.ActorDefinitionVersion.SupportState supportState) { if (supportState == null) { return null; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java index d424646ab00..15a3bb79675 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java @@ -37,6 +37,12 @@ public static io.airbyte.config.NotificationSettings toConfig(final io.airbyte.a if (notification.getSendOnConnectionUpdateActionRequired() != null) { configNotificationSettings.setSendOnConnectionUpdateActionRequired(toConfig(notification.getSendOnConnectionUpdateActionRequired())); } + if (notification.getSendOnBreakingChangeWarning() != null) { + configNotificationSettings.setSendOnBreakingChangeWarning(toConfig(notification.getSendOnBreakingChangeWarning())); + } + if (notification.getSendOnBreakingChangeSyncsDisabled() != null) { + configNotificationSettings.setSendOnBreakingChangeSyncsDisabled(toConfig(notification.getSendOnBreakingChangeSyncsDisabled())); + } return configNotificationSettings; } @@ -90,6 +96,12 @@ public static io.airbyte.api.model.generated.NotificationSettings toApi(final io if (notificationSettings.getSendOnConnectionUpdateActionRequired() != null) { apiNotificationSetings.setSendOnConnectionUpdateActionRequired(toApi(notificationSettings.getSendOnConnectionUpdateActionRequired())); } + if (notificationSettings.getSendOnBreakingChangeWarning() != null) { + apiNotificationSetings.setSendOnBreakingChangeWarning(toApi(notificationSettings.getSendOnBreakingChangeWarning())); + } + if (notificationSettings.getSendOnBreakingChangeSyncsDisabled() != null) { + apiNotificationSetings.setSendOnBreakingChangeSyncsDisabled(toApi(notificationSettings.getSendOnBreakingChangeSyncsDisabled())); + } return apiNotificationSetings; } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java index 5390ea678ab..314b7722a91 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java @@ -4,6 +4,7 @@ package io.airbyte.commons.server.handlers; +import static io.airbyte.commons.server.converters.ApiPojoConverters.toApiSupportLevel; import static io.airbyte.commons.server.converters.ApiPojoConverters.toApiSupportState; import com.google.common.annotations.VisibleForTesting; @@ -19,6 +20,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.validation.json.JsonValidationException; @@ -27,6 +29,7 @@ import java.io.IOException; import java.time.LocalDate; import java.util.List; +import java.util.Objects; /** * ActorDefinitionVersionHandler. Javadocs suppressed because api docs should be used as source of @@ -58,10 +61,10 @@ public ActorDefinitionVersionRead getActorDefinitionVersionForSourceId(final Sou throws JsonValidationException, ConfigNotFoundException, IOException { final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId()); final StandardSourceDefinition sourceDefinition = configRepository.getSourceDefinitionFromSource(sourceConnection.getSourceId()); - final ActorDefinitionVersion actorDefinitionVersion = - actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceConnection.getWorkspaceId(), sourceConnection.getSourceId()); - final boolean isActorDefaultVersion = actorDefinitionVersion.getVersionId().equals(sourceConnection.getDefaultVersionId()); - return createActorDefinitionVersionRead(actorDefinitionVersion, isActorDefaultVersion); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, sourceConnection.getWorkspaceId(), + sourceConnection.getSourceId()); + return createActorDefinitionVersionRead(versionWithOverrideStatus); } public ActorDefinitionVersionRead getActorDefinitionVersionForDestinationId(final DestinationIdRequestBody destinationIdRequestBody) @@ -69,20 +72,24 @@ public ActorDefinitionVersionRead getActorDefinitionVersionForDestinationId(fina final DestinationConnection destinationConnection = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId()); final StandardDestinationDefinition destinationDefinition = configRepository.getDestinationDefinitionFromDestination(destinationConnection.getDestinationId()); - final ActorDefinitionVersion actorDefinitionVersion = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, - destinationConnection.getWorkspaceId(), destinationConnection.getDestinationId()); - final boolean isActorDefaultVersion = actorDefinitionVersion.getVersionId().equals(destinationConnection.getDefaultVersionId()); - return createActorDefinitionVersionRead(actorDefinitionVersion, isActorDefaultVersion); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, + destinationConnection.getWorkspaceId(), destinationConnection.getDestinationId()); + return createActorDefinitionVersionRead(versionWithOverrideStatus); } @VisibleForTesting - ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitionVersion actorDefinitionVersion, final boolean isActorDefault) + ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus) throws IOException { + final ActorDefinitionVersion actorDefinitionVersion = versionWithOverrideStatus.actorDefinitionVersion(); final ActorDefinitionVersionRead advRead = new ActorDefinitionVersionRead() .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(Objects.requireNonNullElse(actorDefinitionVersion.getSupportsDbt(), false)) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(actorDefinitionVersion.getNormalizationConfig())) .supportState(toApiSupportState(actorDefinitionVersion.getSupportState())) - .isActorDefaultVersion(isActorDefault); + .supportLevel(toApiSupportLevel(actorDefinitionVersion.getSupportLevel())) + .isOverrideApplied(versionWithOverrideStatus.isOverrideApplied()); final List breakingChanges = configRepository.listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/AttemptHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/AttemptHandler.java index 40f12344404..88930a49721 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/AttemptHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/AttemptHandler.java @@ -6,6 +6,7 @@ import io.airbyte.api.model.generated.AttemptInfoRead; import io.airbyte.api.model.generated.AttemptStats; +import io.airbyte.api.model.generated.CreateNewAttemptNumberResponse; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.SaveAttemptSyncConfigRequestBody; import io.airbyte.api.model.generated.SaveStatsRequestBody; @@ -13,11 +14,18 @@ import io.airbyte.commons.server.converters.ApiPojoConverters; import io.airbyte.commons.server.converters.JobConverter; import io.airbyte.commons.server.errors.IdNotFoundKnownException; +import io.airbyte.commons.server.handlers.helpers.JobCreationAndStatusUpdateHelper; +import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.config.StreamSyncStats; import io.airbyte.config.SyncStats; +import io.airbyte.config.helpers.LogClientSingleton; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.models.Job; +import jakarta.inject.Named; import jakarta.inject.Singleton; import java.io.IOException; +import java.nio.file.Path; import java.util.Optional; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -35,10 +43,29 @@ public class AttemptHandler { private final JobPersistence jobPersistence; private final JobConverter jobConverter; + private final JobCreationAndStatusUpdateHelper jobCreationAndStatusUpdateHelper; + private final Path workspaceRoot; - public AttemptHandler(final JobPersistence jobPersistence, final JobConverter jobConverter) { + public AttemptHandler(final JobPersistence jobPersistence, + final JobConverter jobConverter, + final JobCreationAndStatusUpdateHelper jobCreationAndStatusUpdateHelper, + @Named("workspaceRoot") final Path workspaceRoot) { this.jobPersistence = jobPersistence; this.jobConverter = jobConverter; + this.jobCreationAndStatusUpdateHelper = jobCreationAndStatusUpdateHelper; + this.workspaceRoot = workspaceRoot; + } + + public CreateNewAttemptNumberResponse createNewAttemptNumber(final long jobId) throws IOException { + final Job job = jobPersistence.getJob(jobId); + + final Path jobRoot = TemporalUtils.getJobRoot(workspaceRoot, String.valueOf(jobId), job.getAttemptsCount()); + final Path logFilePath = jobRoot.resolve(LogClientSingleton.LOG_FILENAME); + final int persistedAttemptNumber = jobPersistence.createAttempt(jobId, logFilePath); + jobCreationAndStatusUpdateHelper.emitJobToReleaseStagesMetric(OssMetricsRegistry.ATTEMPT_CREATED_BY_RELEASE_STAGE, job); + jobCreationAndStatusUpdateHelper.emitAttemptCreatedEvent(job, persistedAttemptNumber); + + return new CreateNewAttemptNumberResponse().attemptNumber(persistedAttemptNumber); } public AttemptInfoRead getAttemptForJob(final long jobId, final int attemptNo) throws IOException { diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java index 1abb20be715..42749957f44 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java @@ -28,6 +28,7 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationRead; import io.airbyte.api.model.generated.DestinationSearch; +import io.airbyte.api.model.generated.DestinationSyncMode; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.ListConnectionsForWorkspacesRequestBody; import io.airbyte.api.model.generated.SourceRead; @@ -728,7 +729,19 @@ public Optional getConnectionAirbyteCatalog(final UUID connectio final ActorDefinitionVersion sourceVersion = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceConnection.getWorkspaceId(), connection.getSourceId()); final io.airbyte.protocol.models.AirbyteCatalog jsonCatalog = Jsons.object(catalog.getCatalog(), io.airbyte.protocol.models.AirbyteCatalog.class); - return Optional.of(CatalogConverter.toApi(jsonCatalog, sourceVersion)); + final StandardDestinationDefinition destination = configRepository.getDestinationDefinitionFromConnection(connectionId); + // Note: we're using the workspace from the source to save an extra db request. + final ActorDefinitionVersion destinationVersion = + actorDefinitionVersionHelper.getDestinationVersion(destination, sourceConnection.getWorkspaceId()); + final List supportedDestinationSyncModes = + Enums.convertListTo(destinationVersion.getSpec().getSupportedDestinationSyncModes(), DestinationSyncMode.class); + final var convertedCatalog = Optional.of(CatalogConverter.toApi(jsonCatalog, sourceVersion)); + if (convertedCatalog.isPresent()) { + convertedCatalog.get().getStreams().forEach((streamAndConfiguration) -> { + CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfiguration, supportedDestinationSyncModes); + }); + } + return convertedCatalog; } public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java index 910783ebf5f..5b8b287b5c7 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java @@ -28,6 +28,7 @@ import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -220,9 +221,10 @@ private UUID createActorDefinition(final String name, final UUID workspaceId, fi .withSpec(connectorSpecification) .withProtocolVersion(DEFAULT_AIRBYTE_PROTOCOL_VERSION.serialize()) .withReleaseStage(ReleaseStage.CUSTOM) + .withSupportLevel(SupportLevel.NONE) .withDocumentationUrl(connectorSpecification.getDocumentationUrl().toString()); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(source, defaultVersion, workspaceId, ScopeType.WORKSPACE); + configRepository.writeCustomConnectorMetadata(source, defaultVersion, workspaceId, ScopeType.WORKSPACE); configRepository.writeActorDefinitionConfigInjectionForPath(manifestInjector.createConfigInjection(source.getSourceDefinitionId(), manifest)); return source.getSourceDefinitionId(); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorDefinitionSpecificationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorDefinitionSpecificationHandler.java new file mode 100644 index 00000000000..cc43095ae6e --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorDefinitionSpecificationHandler.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers; + +import io.airbyte.api.model.generated.AdvancedAuth; +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; +import io.airbyte.api.model.generated.DestinationIdRequestBody; +import io.airbyte.api.model.generated.DestinationSyncMode; +import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; +import io.airbyte.api.model.generated.SourceIdRequestBody; +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.server.converters.JobConverter; +import io.airbyte.commons.server.converters.OauthModelConverter; +import io.airbyte.commons.server.scheduler.SynchronousJobMetadata; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.JobConfig; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import jakarta.inject.Singleton; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +/** + * This class is responsible for getting the specification for a given connector. It is used by the + * {@link io.airbyte.commons.server.handlers.SchedulerHandler} to get the specification for a + * connector. + */ +@Singleton +public class ConnectorDefinitionSpecificationHandler { + + private final ConfigRepository configRepository; + private final ActorDefinitionVersionHelper actorDefinitionVersionHelper; + private final JobConverter jobConverter; + + public ConnectorDefinitionSpecificationHandler(final ConfigRepository configRepository, + final ActorDefinitionVersionHelper actorDefinitionVersionHelper, + final JobConverter jobConverter) { + this.configRepository = configRepository; + this.actorDefinitionVersionHelper = actorDefinitionVersionHelper; + this.jobConverter = jobConverter; + } + + /** + * Get the specification for a given source. + * + * @param sourceIdRequestBody - the id of the source to get the specification for. + * @return the specification for the source. + * @throws JsonValidationException - if the specification is invalid. + * @throws ConfigNotFoundException - if the source does not exist. + * @throws IOException - if there is an error reading the specification. + */ + public SourceDefinitionSpecificationRead getSpecificationForSourceId(final SourceIdRequestBody sourceIdRequestBody) + throws JsonValidationException, ConfigNotFoundException, IOException { + final SourceConnection source = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId()); + final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); + final ActorDefinitionVersion sourceVersion = + actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, source.getWorkspaceId(), sourceIdRequestBody.getSourceId()); + final io.airbyte.protocol.models.ConnectorSpecification spec = sourceVersion.getSpec(); + + return getSourceSpecificationRead(sourceDefinition, spec); + } + + /** + * Get the definition specification for a given source. + * + * @param sourceDefinitionIdWithWorkspaceId - the id of the source to get the specification for. + * @return the specification for the source. + * @throws JsonValidationException - if the specification is invalid. + * @throws ConfigNotFoundException - if the source does not exist. + * @throws IOException - if there is an error reading the specification. + */ + public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) + throws ConfigNotFoundException, IOException, JsonValidationException { + final UUID sourceDefinitionId = sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId(); + final StandardSourceDefinition source = configRepository.getStandardSourceDefinition(sourceDefinitionId); + final ActorDefinitionVersion sourceVersion = + actorDefinitionVersionHelper.getSourceVersion(source, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); + final io.airbyte.protocol.models.ConnectorSpecification spec = sourceVersion.getSpec(); + + return getSourceSpecificationRead(source, spec); + } + + /** + * Get the specification for a given destination. + * + * @param destinationIdRequestBody - the id of the destination to get the specification for. + * @return the specification for the destination. + * @throws JsonValidationException - if the specification is invalid. + * @throws ConfigNotFoundException - if the destination does not exist. + * @throws IOException - if there is an error reading the specification. + */ + public DestinationDefinitionSpecificationRead getSpecificationForDestinationId(final DestinationIdRequestBody destinationIdRequestBody) + throws JsonValidationException, ConfigNotFoundException, IOException { + final DestinationConnection destination = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId()); + final StandardDestinationDefinition destinationDefinition = + configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId()); + final ActorDefinitionVersion destinationVersion = + actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, destination.getWorkspaceId(), + destinationIdRequestBody.getDestinationId()); + final io.airbyte.protocol.models.ConnectorSpecification spec = destinationVersion.getSpec(); + return getDestinationSpecificationRead(destinationDefinition, spec); + } + + /** + * Get the definition specification for a given destination. + * + * @param destinationDefinitionIdWithWorkspaceId - the id of the destination to get the + * specification for. + * @return the specification for the destination. + * @throws JsonValidationException - if the specification is invalid. + * @throws ConfigNotFoundException - if the destination does not exist. + * @throws IOException - if there is an error reading the specification. + */ + @SuppressWarnings("LineLength") + public DestinationDefinitionSpecificationRead getDestinationSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) + throws ConfigNotFoundException, IOException, JsonValidationException { + final UUID destinationDefinitionId = destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId(); + final StandardDestinationDefinition destination = configRepository.getStandardDestinationDefinition(destinationDefinitionId); + final ActorDefinitionVersion destinationVersion = + actorDefinitionVersionHelper.getDestinationVersion(destination, destinationDefinitionIdWithWorkspaceId.getWorkspaceId()); + final io.airbyte.protocol.models.ConnectorSpecification spec = destinationVersion.getSpec(); + + return getDestinationSpecificationRead(destination, spec); + } + + private SourceDefinitionSpecificationRead getSourceSpecificationRead(final StandardSourceDefinition sourceDefinition, + final io.airbyte.protocol.models.ConnectorSpecification spec) { + final SourceDefinitionSpecificationRead specRead = new SourceDefinitionSpecificationRead() + .jobInfo(jobConverter.getSynchronousJobRead(SynchronousJobMetadata.mock(JobConfig.ConfigType.GET_SPEC))) + .connectionSpecification(spec.getConnectionSpecification()) + .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()); + + if (spec.getDocumentationUrl() != null) { + specRead.documentationUrl(spec.getDocumentationUrl().toString()); + } + + final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); + advancedAuth.ifPresent(specRead::setAdvancedAuth); + + return specRead; + } + + private DestinationDefinitionSpecificationRead getDestinationSpecificationRead(final StandardDestinationDefinition destinationDefinition, + final io.airbyte.protocol.models.ConnectorSpecification spec) { + final DestinationDefinitionSpecificationRead specRead = new DestinationDefinitionSpecificationRead() + .jobInfo(jobConverter.getSynchronousJobRead(SynchronousJobMetadata.mock(JobConfig.ConfigType.GET_SPEC))) + .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class)) + .connectionSpecification(spec.getConnectionSpecification()) + .documentationUrl(spec.getDocumentationUrl().toString()) + .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()); + + final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); + advancedAuth.ifPresent(specRead::setAdvancedAuth); + + return specRead; + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index b75ca9b54d8..300584d2148 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -35,12 +35,13 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; import io.airbyte.validation.json.JsonValidationException; import jakarta.inject.Singleton; @@ -70,6 +71,7 @@ public class DestinationDefinitionsHandler { private final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private final RemoteDefinitionsProvider remoteDefinitionsProvider; private final DestinationHandler destinationHandler; + private final SupportStateUpdater supportStateUpdater; private final FeatureFlagClient featureFlagClient; @VisibleForTesting @@ -78,12 +80,14 @@ public DestinationDefinitionsHandler(final ConfigRepository configRepository, final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper, final RemoteDefinitionsProvider remoteDefinitionsProvider, final DestinationHandler destinationHandler, + final SupportStateUpdater supportStateUpdater, final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; this.uuidSupplier = uuidSupplier; this.actorDefinitionHandlerHelper = actorDefinitionHandlerHelper; this.remoteDefinitionsProvider = remoteDefinitionsProvider; this.destinationHandler = destinationHandler; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -100,8 +104,10 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe .documentationUrl(new URI(destinationVersion.getDocumentationUrl())) .icon(loadIcon(standardDestinationDefinition.getIcon())) .protocolVersion(destinationVersion.getProtocolVersion()) + .supportLevel(ApiPojoConverters.toApiSupportLevel(destinationVersion.getSupportLevel())) .releaseStage(ApiPojoConverters.toApiReleaseStage(destinationVersion.getReleaseStage())) .releaseDate(ApiPojoConverters.toLocalDate(destinationVersion.getReleaseDate())) + .custom(standardDestinationDefinition.getCustom()) .supportsDbt(Objects.requireNonNullElse(destinationVersion.getSupportsDbt(), false)) .normalizationConfig( ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(destinationVersion.getNormalizationConfig())) @@ -245,10 +251,10 @@ public DestinationDefinitionRead createCustomDestinationDefinition(final CustomD // legacy call; todo: remove once we drop workspace_id column if (customDestinationDefinitionCreate.getWorkspaceId() != null) { - configRepository.writeCustomDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(destinationDefinition, actorDefinitionVersion, customDestinationDefinitionCreate.getWorkspaceId(), ScopeType.WORKSPACE); } else { - configRepository.writeCustomDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(destinationDefinition, actorDefinitionVersion, customDestinationDefinitionCreate.getScopeId(), ScopeType.fromValue(customDestinationDefinitionCreate.getScopeType().toString())); } @@ -279,10 +285,12 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe final List breakingChangesForDef = actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.DESTINATION); - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestination, newVersion, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestination, newVersion, breakingChangesForDef); - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + final StandardDestinationDefinition updatedDestinationDefinition = configRepository + .getStandardDestinationDefinition(destinationDefinitionUpdate.getDestinationDefinitionId()); + supportStateUpdater.updateSupportStatesForDestinationDefinition(updatedDestinationDefinition); } return buildDestinationDefinitionRead(newDestination, newVersion); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java new file mode 100644 index 00000000000..85c88a8b325 --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers; + +import io.airbyte.api.model.generated.AuthConfiguration; +import io.airbyte.api.model.generated.InstanceConfigurationResponse; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.api.model.generated.WorkspaceUpdate; +import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.license.ActiveAirbyteLicense; +import io.airbyte.config.Configs.AirbyteEdition; +import io.airbyte.config.Organization; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.User; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; +import io.airbyte.validation.json.JsonValidationException; +import io.micronaut.context.annotation.Value; +import jakarta.inject.Singleton; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; + +/** + * InstanceConfigurationHandler. Javadocs suppressed because api docs should be used as source of + * truth. + */ +@SuppressWarnings("MissingJavadocMethod") +@Slf4j +@Singleton +public class InstanceConfigurationHandler { + + private final String webappUrl; + private final AirbyteEdition airbyteEdition; + private final Optional airbyteKeycloakConfiguration; + private final Optional activeAirbyteLicense; + private final WorkspacePersistence workspacePersistence; + private final WorkspacesHandler workspacesHandler; + private final UserPersistence userPersistence; + private final OrganizationPersistence organizationPersistence; + + // the injected webapp-url value defaults to `null` to preserve backwards compatibility. + // TODO remove the default value once configurations are standardized to always include a + // webapp-url. + public InstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final String webappUrl, + final AirbyteEdition airbyteEdition, + final Optional airbyteKeycloakConfiguration, + final Optional activeAirbyteLicense, + final WorkspacePersistence workspacePersistence, + final WorkspacesHandler workspacesHandler, + final UserPersistence userPersistence, + final OrganizationPersistence organizationPersistence) { + this.webappUrl = webappUrl; + this.airbyteEdition = airbyteEdition; + this.airbyteKeycloakConfiguration = airbyteKeycloakConfiguration; + this.activeAirbyteLicense = activeAirbyteLicense; + this.workspacePersistence = workspacePersistence; + this.workspacesHandler = workspacesHandler; + this.userPersistence = userPersistence; + this.organizationPersistence = organizationPersistence; + } + + public InstanceConfigurationResponse getInstanceConfiguration() throws IOException { + final UUID defaultOrganizationId = getDefaultOrganizationId(); + final StandardWorkspace defaultWorkspace = getDefaultWorkspace(defaultOrganizationId); + + return new InstanceConfigurationResponse() + .webappUrl(webappUrl) + .edition(Enums.convertTo(airbyteEdition, EditionEnum.class)) + .licenseType(getLicenseType()) + .auth(getAuthConfiguration()) + .initialSetupComplete(defaultWorkspace.getInitialSetupComplete()) + .defaultUserId(getDefaultUserId()) + .defaultOrganizationId(defaultOrganizationId) + .defaultWorkspaceId(defaultWorkspace.getWorkspaceId()); + } + + public InstanceConfigurationResponse setupInstanceConfiguration(final InstanceConfigurationSetupRequestBody requestBody) + throws IOException, JsonValidationException, ConfigNotFoundException { + + final UUID defaultOrganizationId = getDefaultOrganizationId(); + final StandardWorkspace defaultWorkspace = getDefaultWorkspace(defaultOrganizationId); + + // Update the default organization and user with the provided information + updateDefaultOrganization(requestBody); + updateDefaultUser(requestBody); + + // Update the underlying workspace to mark the initial setup as complete + workspacesHandler.updateWorkspace(new WorkspaceUpdate() + .workspaceId(defaultWorkspace.getWorkspaceId()) + .email(requestBody.getEmail()) + .displaySetupWizard(requestBody.getDisplaySetupWizard()) + .anonymousDataCollection(requestBody.getAnonymousDataCollection()) + .initialSetupComplete(requestBody.getInitialSetupComplete())); + + // Return the updated instance configuration + return getInstanceConfiguration(); + } + + private LicenseTypeEnum getLicenseType() { + if (airbyteEdition.equals(AirbyteEdition.PRO) && activeAirbyteLicense.isPresent()) { + return Enums.convertTo(activeAirbyteLicense.get().getLicenseType(), LicenseTypeEnum.class); + } else { + return null; + } + } + + private AuthConfiguration getAuthConfiguration() { + if (airbyteEdition.equals(AirbyteEdition.PRO) && airbyteKeycloakConfiguration.isPresent()) { + return new AuthConfiguration() + .clientId(airbyteKeycloakConfiguration.get().getWebClientId()) + .defaultRealm(airbyteKeycloakConfiguration.get().getAirbyteRealm()); + } else { + return null; + } + } + + private UUID getDefaultUserId() throws IOException { + return userPersistence.getDefaultUser().orElseThrow(() -> new IllegalStateException("Default user does not exist.")).getUserId(); + } + + private void updateDefaultUser(final InstanceConfigurationSetupRequestBody requestBody) throws IOException { + final User defaultUser = userPersistence.getDefaultUser().orElseThrow(() -> new IllegalStateException("Default user does not exist.")); + // email is a required request property, so always set it. + defaultUser.setEmail(requestBody.getEmail()); + + // name is currently optional, so only set it if it is provided. + if (requestBody.getUserName() != null) { + defaultUser.setName(requestBody.getUserName()); + } + + userPersistence.writeUser(defaultUser); + } + + private UUID getDefaultOrganizationId() throws IOException { + return organizationPersistence.getDefaultOrganization() + .orElseThrow(() -> new IllegalStateException("Default organization does not exist.")) + .getOrganizationId(); + } + + private void updateDefaultOrganization(final InstanceConfigurationSetupRequestBody requestBody) throws IOException { + final Organization defaultOrganization = + organizationPersistence.getDefaultOrganization().orElseThrow(() -> new IllegalStateException("Default organization does not exist.")); + + // email is a required request property, so always set it. + defaultOrganization.setEmail(requestBody.getEmail()); + + // name is currently optional, so only set it if it is provided. + if (requestBody.getOrganizationName() != null) { + defaultOrganization.setName(requestBody.getOrganizationName()); + } + + organizationPersistence.updateOrganization(defaultOrganization); + } + + // Historically, instance setup for an OSS installation of Airbyte was stored on the one and only + // workspace that was created for the instance. Now that OSS supports multiple workspaces, we + // use the default Organization ID to select a workspace to use for instance setup. This is a hack. + // TODO persist instance configuration to a separate resource, rather than using a workspace. + private StandardWorkspace getDefaultWorkspace(final UUID organizationId) throws IOException { + return workspacePersistence.getDefaultWorkspaceForOrganization(organizationId); + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index b8c7e4f6040..9ff5cc780d9 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -14,7 +14,6 @@ import com.google.common.collect.Lists; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import io.airbyte.api.model.generated.AdvancedAuth; import io.airbyte.api.model.generated.CatalogDiff; import io.airbyte.api.model.generated.CheckConnectionRead; import io.airbyte.api.model.generated.CheckConnectionRead.StatusEnum; @@ -27,11 +26,9 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationCoreConfig; import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; -import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; import io.airbyte.api.model.generated.DestinationIdRequestBody; import io.airbyte.api.model.generated.DestinationSyncMode; import io.airbyte.api.model.generated.DestinationUpdate; -import io.airbyte.api.model.generated.FieldTransform; import io.airbyte.api.model.generated.JobConfigType; import io.airbyte.api.model.generated.JobCreate; import io.airbyte.api.model.generated.JobIdRequestBody; @@ -40,26 +37,22 @@ import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.SourceAutoPropagateChange; import io.airbyte.api.model.generated.SourceCoreConfig; -import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; -import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; import io.airbyte.api.model.generated.SourceDiscoverSchemaRead; import io.airbyte.api.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.api.model.generated.SourceIdRequestBody; import io.airbyte.api.model.generated.SourceUpdate; import io.airbyte.api.model.generated.StreamTransform; -import io.airbyte.api.model.generated.StreamTransform.TransformTypeEnum; import io.airbyte.api.model.generated.SynchronousJobRead; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.converters.ConfigurationUpdate; import io.airbyte.commons.server.converters.JobConverter; -import io.airbyte.commons.server.converters.OauthModelConverter; import io.airbyte.commons.server.errors.ValueConflictKnownException; +import io.airbyte.commons.server.handlers.helpers.AutoPropagateSchemaChangeHelper; import io.airbyte.commons.server.handlers.helpers.CatalogConverter; import io.airbyte.commons.server.handlers.helpers.JobCreationAndStatusUpdateHelper; import io.airbyte.commons.server.scheduler.EventRunner; -import io.airbyte.commons.server.scheduler.SynchronousJobMetadata; import io.airbyte.commons.server.scheduler.SynchronousResponse; import io.airbyte.commons.server.scheduler.SynchronousSchedulerClient; import io.airbyte.commons.temporal.ErrorCode; @@ -69,7 +62,6 @@ import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.DestinationConnection; -import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobTypeResourceLimit.JobType; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.SourceConnection; @@ -102,7 +94,6 @@ import io.airbyte.persistence.job.models.Job; import io.airbyte.persistence.job.tracker.JobTracker; import io.airbyte.protocol.models.AirbyteCatalog; -import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; @@ -150,6 +141,7 @@ public class SchedulerHandler { private final JobCreator jobCreator; private final SyncJobFactory jobFactory; private final JobCreationAndStatusUpdateHelper jobCreationAndStatusUpdateHelper; + private final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler; // TODO: Convert to be fully using micronaut public SchedulerHandler(final ConfigRepository configRepository, @@ -170,7 +162,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final JobCreator jobCreator, final SyncJobFactory jobFactory, final JobNotifier jobNotifier, - final JobTracker jobTracker) { + final JobTracker jobTracker, + final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler) { this( configRepository, secretsRepositoryWriter, @@ -190,7 +183,8 @@ public SchedulerHandler(final ConfigRepository configRepository, jobCreator, jobFactory, jobNotifier, - jobTracker); + jobTracker, + connectorDefinitionSpecificationHandler); } @VisibleForTesting @@ -212,7 +206,8 @@ public SchedulerHandler(final ConfigRepository configRepository, final JobCreator jobCreator, final SyncJobFactory jobFactory, final JobNotifier jobNotifier, - final JobTracker jobTracker) { + final JobTracker jobTracker, + final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler) { this.configRepository = configRepository; this.secretsRepositoryWriter = secretsRepositoryWriter; this.synchronousSchedulerClient = synchronousSchedulerClient; @@ -230,6 +225,7 @@ public SchedulerHandler(final ConfigRepository configRepository, this.oAuthConfigSupplier = oAuthConfigSupplier; this.jobCreator = jobCreator; this.jobFactory = jobFactory; + this.connectorDefinitionSpecificationHandler = connectorDefinitionSpecificationHandler; this.jobCreationAndStatusUpdateHelper = new JobCreationAndStatusUpdateHelper( jobPersistence, configRepository, @@ -448,6 +444,13 @@ public void applySchemaChangeForSource(final SourceAutoPropagateChange sourceAut final ConnectionUpdate updateObject = new ConnectionUpdate().connectionId(connectionRead.getConnectionId()); + final UUID destinationDefinitionId = + configRepository.getDestinationDefinitionFromConnection(connectionRead.getConnectionId()).getDestinationDefinitionId(); + final var supportedDestinationSyncModes = + connectorDefinitionSpecificationHandler + .getDestinationSpecification(new DestinationDefinitionIdWithWorkspaceId().destinationDefinitionId(destinationDefinitionId) + .workspaceId(sourceAutoPropagateChange.getWorkspaceId())) + .getSupportedDestinationSyncModes(); if (shouldAutoPropagate(diff, sourceAutoPropagateChange.getWorkspaceId(), connectionRead)) { applySchemaChange(updateObject.getConnectionId(), @@ -457,7 +460,7 @@ public void applySchemaChangeForSource(final SourceAutoPropagateChange sourceAut sourceAutoPropagateChange.getCatalog(), diff.getTransforms(), sourceAutoPropagateChange.getCatalogId(), - connectionRead.getNonBreakingChangesPreference()); + connectionRead.getNonBreakingChangesPreference(), supportedDestinationSyncModes); connectionsHandler.updateConnection(updateObject); LOGGER.info("Propagating changes for connectionId: '{}', new catalogId '{}'", connectionRead.getConnectionId(), sourceAutoPropagateChange.getCatalogId()); @@ -510,85 +513,6 @@ private SourceDiscoverSchemaRead retrieveDiscoveredSchema(final SynchronousRespo return sourceDiscoverSchemaRead; } - public SourceDefinitionSpecificationRead getSpecificationForSourceId(final SourceIdRequestBody sourceIdRequestBody) - throws JsonValidationException, ConfigNotFoundException, IOException { - final SourceConnection source = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId()); - final StandardSourceDefinition sourceDefinition = configRepository.getStandardSourceDefinition(source.getSourceDefinitionId()); - final ActorDefinitionVersion sourceVersion = - actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, source.getWorkspaceId(), sourceIdRequestBody.getSourceId()); - final ConnectorSpecification spec = sourceVersion.getSpec(); - - return getSourceSpecificationRead(sourceDefinition, spec); - } - - public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) - throws ConfigNotFoundException, IOException, JsonValidationException { - final UUID sourceDefinitionId = sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId(); - final StandardSourceDefinition source = configRepository.getStandardSourceDefinition(sourceDefinitionId); - final ActorDefinitionVersion sourceVersion = - actorDefinitionVersionHelper.getSourceVersion(source, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); - final ConnectorSpecification spec = sourceVersion.getSpec(); - - return getSourceSpecificationRead(source, spec); - } - - private SourceDefinitionSpecificationRead getSourceSpecificationRead(final StandardSourceDefinition sourceDefinition, - final ConnectorSpecification spec) { - final SourceDefinitionSpecificationRead specRead = new SourceDefinitionSpecificationRead() - .jobInfo(jobConverter.getSynchronousJobRead(SynchronousJobMetadata.mock(ConfigType.GET_SPEC))) - .connectionSpecification(spec.getConnectionSpecification()) - .sourceDefinitionId(sourceDefinition.getSourceDefinitionId()); - - if (spec.getDocumentationUrl() != null) { - specRead.documentationUrl(spec.getDocumentationUrl().toString()); - } - - final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); - advancedAuth.ifPresent(specRead::setAdvancedAuth); - - return specRead; - } - - public DestinationDefinitionSpecificationRead getSpecificationForDestinationId(final DestinationIdRequestBody destinationIdRequestBody) - throws JsonValidationException, ConfigNotFoundException, IOException { - final DestinationConnection destination = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId()); - final StandardDestinationDefinition destinationDefinition = - configRepository.getStandardDestinationDefinition(destination.getDestinationDefinitionId()); - final ActorDefinitionVersion destinationVersion = - actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, destination.getWorkspaceId(), - destinationIdRequestBody.getDestinationId()); - final ConnectorSpecification spec = destinationVersion.getSpec(); - return getDestinationSpecificationRead(destinationDefinition, spec); - } - - @SuppressWarnings("LineLength") - public DestinationDefinitionSpecificationRead getDestinationSpecification( - final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) - throws ConfigNotFoundException, IOException, JsonValidationException { - final UUID destinationDefinitionId = destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId(); - final StandardDestinationDefinition destination = configRepository.getStandardDestinationDefinition(destinationDefinitionId); - final ActorDefinitionVersion destinationVersion = - actorDefinitionVersionHelper.getDestinationVersion(destination, destinationDefinitionIdWithWorkspaceId.getWorkspaceId()); - final ConnectorSpecification spec = destinationVersion.getSpec(); - - return getDestinationSpecificationRead(destination, spec); - } - - private DestinationDefinitionSpecificationRead getDestinationSpecificationRead(final StandardDestinationDefinition destinationDefinition, - final ConnectorSpecification spec) { - final DestinationDefinitionSpecificationRead specRead = new DestinationDefinitionSpecificationRead() - .jobInfo(jobConverter.getSynchronousJobRead(SynchronousJobMetadata.mock(ConfigType.GET_SPEC))) - .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class)) - .connectionSpecification(spec.getConnectionSpecification()) - .documentationUrl(spec.getDocumentationUrl().toString()) - .destinationDefinitionId(destinationDefinition.getDestinationDefinitionId()); - - final Optional advancedAuth = OauthModelConverter.getAdvancedAuth(spec); - advancedAuth.ifPresent(specRead::setAdvancedAuth); - - return specRead; - } - public JobInfoRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) throws IOException, JsonValidationException, ConfigNotFoundException { return submitManualSyncToWorker(connectionIdRequestBody.getConnectionId()); @@ -631,7 +555,13 @@ public JobInfoRead createJob(final JobCreate jobCreate) throws JsonValidationExc final List standardSyncOperations = Lists.newArrayList(); for (final var operationId : standardSync.getOperationIds()) { final StandardSyncOperation standardSyncOperation = configRepository.getStandardSyncOperation(operationId); - standardSyncOperations.add(standardSyncOperation); + // NOTE: we must run normalization operations during resets, because we rely on them to clear the + // normalized tables. However, we don't want to run other operations (dbt, webhook) because those + // are meant to transform the data after the sync but there's no data to transform. Webhook + // operations particularly will fail because we don't populate some required config during resets. + if (StandardSyncOperation.OperatorType.NORMALIZATION.equals(standardSyncOperation.getOperatorType())) { + standardSyncOperations.add(standardSyncOperation); + } } final Optional jobIdOptional = @@ -685,7 +615,7 @@ private void generateCatalogDiffsAndDisableConnectionsIfNeeded(final SourceDisco final CatalogDiff diff = connectionsHandler.getDiff(catalogUsedToMakeConfiguredCatalog.orElse(currentAirbyteCatalog), discoveredSchema.getCatalog(), CatalogConverter.toConfiguredProtocol(currentAirbyteCatalog)); - final boolean containsBreakingChange = containsBreakingChange(diff); + final boolean containsBreakingChange = AutoPropagateSchemaChangeHelper.containsBreakingChange(diff); if (containsBreakingChange) { MetricClientFactory.getMetricClient().count(OssMetricsRegistry.BREAKING_SCHEMA_CHANGE_DETECTED, 1, @@ -721,7 +651,7 @@ private void generateCatalogDiffsAndDisableConnectionsIfNeeded(final SourceDisco private boolean shouldAutoPropagate(final CatalogDiff diff, final UUID workspaceId, final ConnectionRead connectionRead) { final boolean hasDiff = !diff.getTransforms().isEmpty(); - final boolean nonBreakingChange = !containsBreakingChange(diff); + final boolean nonBreakingChange = !AutoPropagateSchemaChangeHelper.containsBreakingChange(diff); final boolean autoPropagationIsEnabledForWorkspace = featureFlagClient.boolVariation(AutoPropagateSchema.INSTANCE, new Workspace(workspaceId)); final boolean autoPropagationIsEnabledForConnection = connectionRead.getNonBreakingChangesPreference() != null @@ -737,7 +667,8 @@ private void applySchemaChange(final UUID connectionId, final io.airbyte.api.model.generated.AirbyteCatalog newCatalog, final List transformations, final UUID sourceCatalogId, - final NonBreakingChangesPreference nonBreakingChangesPreference) { + final NonBreakingChangesPreference nonBreakingChangesPreference, + final List supportedDestinationSyncModes) { MetricClientFactory.getMetricClient().count(OssMetricsRegistry.SCHEMA_CHANGE_AUTO_PROPAGATED, 1, new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); final io.airbyte.api.model.generated.AirbyteCatalog catalog = getUpdatedSchema( @@ -745,7 +676,8 @@ private void applySchemaChange(final UUID connectionId, newCatalog, transformations, nonBreakingChangesPreference, - featureFlagClient, workspaceId); + supportedDestinationSyncModes, + featureFlagClient, workspaceId).catalog(); updateObject.setSyncCatalog(catalog); updateObject.setSourceCatalogId(sourceCatalogId); } @@ -842,20 +774,4 @@ private JobInfoRead readJobFromResult(final ManualOperationResult manualOperatio return jobConverter.getJobInfoRead(job); } - @VisibleForTesting - boolean containsBreakingChange(final CatalogDiff diff) { - for (final StreamTransform streamTransform : diff.getTransforms()) { - if (streamTransform.getTransformType() != TransformTypeEnum.UPDATE_STREAM) { - continue; - } - - final boolean anyBreakingFieldTransforms = streamTransform.getUpdateStream().stream().anyMatch(FieldTransform::getBreaking); - if (anyBreakingFieldTransforms) { - return true; - } - } - - return false; - } - } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index 753bc26e169..1077e0f240b 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -36,11 +36,12 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.Workspace; import io.airbyte.validation.json.JsonValidationException; @@ -70,6 +71,7 @@ public class SourceDefinitionsHandler { private final RemoteDefinitionsProvider remoteDefinitionsProvider; private final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private final SourceHandler sourceHandler; + private final SupportStateUpdater supportStateUpdater; private final FeatureFlagClient featureFlagClient; @Inject @@ -78,12 +80,14 @@ public SourceDefinitionsHandler(final ConfigRepository configRepository, final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper, final RemoteDefinitionsProvider remoteDefinitionsProvider, final SourceHandler sourceHandler, + final SupportStateUpdater supportStateUpdater, final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; this.uuidSupplier = uuidSupplier; this.actorDefinitionHandlerHelper = actorDefinitionHandlerHelper; this.remoteDefinitionsProvider = remoteDefinitionsProvider; this.sourceHandler = sourceHandler; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -100,8 +104,10 @@ static SourceDefinitionRead buildSourceDefinitionRead(final StandardSourceDefini .documentationUrl(new URI(sourceVersion.getDocumentationUrl())) .icon(loadIcon(standardSourceDefinition.getIcon())) .protocolVersion(sourceVersion.getProtocolVersion()) + .supportLevel(ApiPojoConverters.toApiSupportLevel(sourceVersion.getSupportLevel())) .releaseStage(ApiPojoConverters.toApiReleaseStage(sourceVersion.getReleaseStage())) .releaseDate(ApiPojoConverters.toLocalDate(sourceVersion.getReleaseDate())) + .custom(standardSourceDefinition.getCustom()) .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardSourceDefinition.getResourceRequirements())) .maxSecondsBetweenMessages(standardSourceDefinition.getMaxSecondsBetweenMessages()); @@ -245,10 +251,10 @@ public SourceDefinitionRead createCustomSourceDefinition(final CustomSourceDefin // legacy call; todo: remove once we drop workspace_id column if (customSourceDefinitionCreate.getWorkspaceId() != null) { - configRepository.writeCustomSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(sourceDefinition, actorDefinitionVersion, customSourceDefinitionCreate.getWorkspaceId(), ScopeType.WORKSPACE); } else { - configRepository.writeCustomSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(sourceDefinition, actorDefinitionVersion, customSourceDefinitionCreate.getScopeId(), ScopeType.fromValue(customSourceDefinitionCreate.getScopeType().toString())); } @@ -279,10 +285,11 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate currentVersion, ActorType.SOURCE, sourceDefinitionUpdate.getDockerImageTag(), currentSourceDefinition.getCustom()); final List breakingChangesForDef = actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.SOURCE); - configRepository.writeSourceDefinitionAndDefaultVersion(newSource, newVersion, breakingChangesForDef); + configRepository.writeConnectorMetadata(newSource, newVersion, breakingChangesForDef); - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + final StandardSourceDefinition updatedSourceDefinition = configRepository.getStandardSourceDefinition(newSource.getSourceDefinitionId()); + supportStateUpdater.updateSupportStatesForSourceDefinition(updatedSourceDefinition); } return buildSourceDefinitionRead(newSource, newVersion); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java index 68de335ff1d..cc7710a36d8 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java @@ -40,9 +40,7 @@ import io.airbyte.config.persistence.SecretsRepositoryWriter; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; -import io.airbyte.featureflag.CanonicalCatalogSchema; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.Source; import io.airbyte.persistence.job.factory.OAuthConfigSupplier; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; @@ -379,30 +377,11 @@ public void deleteSource(final SourceRead source) public DiscoverCatalogResult writeDiscoverCatalogResult(final SourceDiscoverSchemaWriteRequestBody request) throws JsonValidationException, IOException { final AirbyteCatalog persistenceCatalog = CatalogConverter.toProtocol(request.getCatalog()); - final UUID catalogId; - - if (shouldWriteCanonicalActorCatalog(request)) { - catalogId = writeCanonicalActorCatalog(persistenceCatalog, request); - } else { - catalogId = writeActorCatalog(persistenceCatalog, request); - } + final UUID catalogId = writeActorCatalog(persistenceCatalog, request); return new DiscoverCatalogResult().catalogId(catalogId); } - private boolean shouldWriteCanonicalActorCatalog(final SourceDiscoverSchemaWriteRequestBody request) { - return request.getSourceId() != null && featureFlagClient.boolVariation(CanonicalCatalogSchema.INSTANCE, new Source(request.getSourceId())); - } - - private UUID writeCanonicalActorCatalog(final AirbyteCatalog persistenceCatalog, final SourceDiscoverSchemaWriteRequestBody request) - throws IOException { - return configRepository.writeCanonicalActorCatalogFetchEvent( - persistenceCatalog, - request.getSourceId(), - request.getConnectorVersion(), - request.getConfigurationHash()); - } - private UUID writeActorCatalog(final AirbyteCatalog persistenceCatalog, final SourceDiscoverSchemaWriteRequestBody request) throws IOException { return configRepository.writeActorCatalogFetchEvent( persistenceCatalog, diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java index 0bbe88fe9c9..f401df1a61c 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java @@ -15,6 +15,8 @@ import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserStatus; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoRead; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserRead; import io.airbyte.api.model.generated.WorkspaceUserReadList; @@ -44,7 +46,7 @@ /** * UserHandler, provides basic CRUD operation access for users. Some are migrated from Cloud - * UserHandler {@link io.airbyte.cloud.server.handlers.UserHandler}. + * UserHandler. */ @SuppressWarnings({"MissingJavadocMethod"}) @Singleton @@ -253,6 +255,11 @@ public WorkspaceUserReadList listUsersInWorkspace(final WorkspaceIdRequestBody w return buildWorkspaceUserReadList(userPermissions, workspaceId); } + public UserWithPermissionInfoReadList listInstanceAdminUsers() throws IOException { + final List userPermissions = permissionPersistence.listInstanceAdminUsers(); + return buildUserWithPermissionInfoReadList(userPermissions); + } + private Map collectUserPermissionToMap(final List userPermissions) { return userPermissions.stream() .collect(Collectors.toMap( @@ -278,6 +285,18 @@ private WorkspaceUserReadList buildWorkspaceUserReadList(final List userPermissions) { + return new UserWithPermissionInfoReadList().users( + collectUserPermissionToMap(userPermissions) + .entrySet().stream() + .map((Entry entry) -> new UserWithPermissionInfoRead() + .userId(entry.getKey().getUserId()) + .email(entry.getKey().getEmail()) + .name(entry.getKey().getName()) + .permissionId(entry.getValue().getPermissionId())) + .collect(Collectors.toList())); + } + private OrganizationUserReadList buildOrganizationUserReadList(final List userPermissions, final UUID organizationId) { return new OrganizationUserReadList().users(collectUserPermissionToMap(userPermissions) .entrySet().stream() diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java index 88e0d6691f0..7f4243924a3 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java @@ -358,14 +358,14 @@ public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnecti final AirbyteCatalog configuredCatalog = connection.getSyncCatalog(); /* * This catalog represents the full catalog that was used to create the configured catalog. It will - * have all streams that were present at the time. It will have no configuration set. + * have all streams that were present at the time. It will have default configuration options set. */ final Optional catalogUsedToMakeConfiguredCatalog = connectionsHandler .getConnectionAirbyteCatalog(webBackendConnectionRequestBody.getConnectionId()); /* - * This catalog represents the full catalog that exists now for the source. It will have no - * configuration set. + * This catalog represents the full catalog that exists now for the source. It will have default + * configuration options set. */ final Optional refreshedCatalog; if (MoreBooleans.isTruthy(webBackendConnectionRequestBody.getWithRefreshedCatalog())) { diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java index fdbc568fce0..2ea424d05a1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java @@ -14,6 +14,7 @@ import io.airbyte.api.model.generated.DestinationRead; import io.airbyte.api.model.generated.Geography; import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; +import io.airbyte.api.model.generated.ListWorkspacesByUserRequestBody; import io.airbyte.api.model.generated.ListWorkspacesInOrganizationRequestBody; import io.airbyte.api.model.generated.NotificationItem; import io.airbyte.api.model.generated.NotificationSettings; @@ -35,9 +36,11 @@ import io.airbyte.commons.server.errors.InternalServerKnownException; import io.airbyte.commons.server.errors.ValueConflictKnownException; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.UserPermission; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.config.persistence.ConfigRepository.ResourcesQueryPaginated; import io.airbyte.config.persistence.PermissionPersistence; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -178,7 +181,9 @@ private NotificationSettings patchNotificationSettingsWithDefaultValue(final Wor .sendOnConnectionUpdate(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) .sendOnConnectionUpdateActionRequired(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) .sendOnSyncDisabled(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) - .sendOnSyncDisabledWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)); + .sendOnSyncDisabledWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)); if (workspaceCreate.getNotificationSettings() != null) { final NotificationSettings inputNotificationSettings = workspaceCreate.getNotificationSettings(); if (inputNotificationSettings.getSendOnSuccess() != null) { @@ -199,6 +204,12 @@ private NotificationSettings patchNotificationSettingsWithDefaultValue(final Wor if (inputNotificationSettings.getSendOnSyncDisabledWarning() != null) { notificationSettings.setSendOnSyncDisabledWarning(inputNotificationSettings.getSendOnSyncDisabledWarning()); } + if (inputNotificationSettings.getSendOnBreakingChangeWarning() != null) { + notificationSettings.setSendOnBreakingChangeWarning(inputNotificationSettings.getSendOnBreakingChangeWarning()); + } + if (inputNotificationSettings.getSendOnBreakingChangeSyncsDisabled() != null) { + notificationSettings.setSendOnBreakingChangeSyncsDisabled(inputNotificationSettings.getSendOnBreakingChangeSyncsDisabled()); + } } return notificationSettings; } @@ -248,8 +259,7 @@ public WorkspaceRead getWorkspace(final WorkspaceIdRequestBody workspaceIdReques } @SuppressWarnings("unused") - public WorkspaceRead getWorkspaceBySlug(final SlugRequestBody slugRequestBody) - throws JsonValidationException, IOException, ConfigNotFoundException { + public WorkspaceRead getWorkspaceBySlug(final SlugRequestBody slugRequestBody) throws IOException, ConfigNotFoundException { // for now we assume there is one workspace and it has a default uuid. final StandardWorkspace workspace = configRepository.getWorkspaceBySlug(slugRequestBody.getSlug(), false); return buildWorkspaceRead(workspace); @@ -260,17 +270,75 @@ public WorkspaceRead getWorkspaceByConnectionId(final ConnectionIdRequestBody co return buildWorkspaceRead(workspace); } - public WorkspaceReadList listWorkspacesInOrganization(final ListWorkspacesInOrganizationRequestBody request) - throws ConfigNotFoundException, IOException { + public WorkspaceReadList listWorkspacesInOrganization(final ListWorkspacesInOrganizationRequestBody request) throws IOException { Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); - final List standardWorkspaces = workspacePersistence - .listWorkspacesByOrganizationId( - new ResourcesByOrganizationQueryPaginated(request.getOrganizationId(), - false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), - keyword) - .stream() - .map(WorkspacesHandler::buildWorkspaceRead) - .collect(Collectors.toList()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByOrganizationIdPaginated( + new ResourcesByOrganizationQueryPaginated(request.getOrganizationId(), + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByOrganizationId(request.getOrganizationId(), false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } + return new WorkspaceReadList().workspaces(standardWorkspaces); + } + + private WorkspaceReadList listWorkspacesByInstanceAdminUser(final ListWorkspacesByUserRequestBody request) throws IOException { + Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByInstanceAdminUserPaginated( + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset(), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByInstanceAdminUser(false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } + return new WorkspaceReadList().workspaces(standardWorkspaces); + } + + public WorkspaceReadList listWorkspacesByUser(final ListWorkspacesByUserRequestBody request) + throws IOException { + // If user has instance_admin permission, list all workspaces. + final UserPermission userInstanceAdminPermission = permissionPersistence.getUserInstanceAdminPermission(request.getUserId()); + if (userInstanceAdminPermission != null) { + return listWorkspacesByInstanceAdminUser(request); + } + // User has no instance_admin permission. + Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByUserIdPaginated( + new ResourcesByUserQueryPaginated(request.getUserId(), + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByUserId(request.getUserId(), false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } return new WorkspaceReadList().workspaces(standardWorkspaces); } @@ -333,7 +401,7 @@ private WorkspaceRead buildWorkspaceReadFromId(final UUID workspaceId) throws Co return buildWorkspaceRead(workspace); } - private String generateUniqueSlug(final String workspaceName) throws JsonValidationException, IOException { + private String generateUniqueSlug(final String workspaceName) throws IOException { final String proposedSlug = slugify.slugify(workspaceName); // todo (cgardens) - this is going to be too expensive once there are too many workspaces. needs to diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java index 03305779cfa..4c3996a0692 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java @@ -83,6 +83,7 @@ public ActorDefinitionVersion defaultDefinitionVersionFromCreate(final String do .withSpec(spec) .withDocumentationUrl(documentationUrl.toString()) .withProtocolVersion(protocolVersion) + .withSupportLevel(io.airbyte.config.SupportLevel.NONE) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM); } @@ -131,6 +132,7 @@ public ActorDefinitionVersion defaultDefinitionVersionFromUpdate(final ActorDefi .withProtocolVersion(protocolVersion) .withReleaseStage(currentVersion.getReleaseStage()) .withReleaseDate(currentVersion.getReleaseDate()) + .withSupportLevel(currentVersion.getSupportLevel()) .withNormalizationConfig(currentVersion.getNormalizationConfig()) .withSupportsDbt(currentVersion.getSupportsDbt()) .withAllowedHosts(currentVersion.getAllowedHosts()); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java index fed93cfde99..480d1598676 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java @@ -7,15 +7,17 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.api.model.generated.AirbyteCatalog; import io.airbyte.api.model.generated.AirbyteStreamAndConfiguration; +import io.airbyte.api.model.generated.CatalogDiff; import io.airbyte.api.model.generated.DestinationSyncMode; +import io.airbyte.api.model.generated.FieldTransform; import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.StreamTransform; -import io.airbyte.api.model.generated.SyncMode; import io.airbyte.commons.json.Jsons; import io.airbyte.featureflag.AutoPropagateNewStreams; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.Workspace; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; @@ -36,6 +38,49 @@ private enum DefaultSyncModeCase { NO_SOURCE_CURSOR } + /** + * Return value for `getUpdatedSchema` method. Returns the updated catalog and a description of the + * changes + */ + public record UpdateSchemaResult(AirbyteCatalog catalog, List changeDescription) {} + + /** + * Generate a summary of the changes that will be applied to the destination if the schema is + * updated. + * + * @param transform the transformation to be applied to the destination + * @return a list of strings describing the changes that will be applied to the destination. + */ + private static String staticFormatDiff(StreamTransform transform) { + switch (transform.getTransformType()) { + case ADD_STREAM -> { + return String.format("Added new stream %s", transform.getStreamDescriptor().getName()); + } + case REMOVE_STREAM -> { + return String.format("Removed stream %s", transform.getStreamDescriptor().getName()); + } + case UPDATE_STREAM -> { + String returnValue = String.format("Modified stream %s", transform.getStreamDescriptor().getName()); + if (transform.getUpdateStream() == null) { + return returnValue; + } + for (FieldTransform fieldTransform : transform.getUpdateStream()) { + String fieldName = String.join(".", fieldTransform.getFieldName()); + switch (fieldTransform.getTransformType()) { + case ADD_FIELD -> returnValue += String.format("Added field: %s,", fieldName); + case REMOVE_FIELD -> returnValue += String.format("Removed field: %s,", fieldName); + case UPDATE_FIELD_SCHEMA -> returnValue += String.format("Field type changed: %s,", fieldName); + default -> throw new NotSupportedException("Not supported transformation."); + } + } + return returnValue; + } + default -> { + return "Unknown Transformation"; + } + } + } + /** * This is auto propagating schema changes, it replaces the stream in the old catalog by using the * ones from the new catalog. The list of transformations contains the information of which stream @@ -47,16 +92,19 @@ private enum DefaultSyncModeCase { * @param nonBreakingChangesPreference User preference for the auto propagation * @return an Airbyte catalog the changes being auto propagated */ - public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, - final AirbyteCatalog newCatalog, - final List transformations, - final NonBreakingChangesPreference nonBreakingChangesPreference, - final FeatureFlagClient featureFlagClient, - final UUID workspaceId) { + public static UpdateSchemaResult getUpdatedSchema(final AirbyteCatalog oldCatalog, + final AirbyteCatalog newCatalog, + final List transformations, + final NonBreakingChangesPreference nonBreakingChangesPreference, + final List supportedDestinationSyncModes, + final FeatureFlagClient featureFlagClient, + final UUID workspaceId) { final AirbyteCatalog copiedOldCatalog = Jsons.clone(oldCatalog); final Map oldCatalogPerStream = extractStreamAndConfigPerStreamDescriptor(copiedOldCatalog); final Map newCatalogPerStream = extractStreamAndConfigPerStreamDescriptor(newCatalog); + List changes = new ArrayList<>(); + transformations.forEach(transformation -> { final StreamDescriptor streamDescriptor = transformation.getStreamDescriptor(); switch (transformation.getTransformType()) { @@ -64,6 +112,7 @@ public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, if (oldCatalogPerStream.containsKey(streamDescriptor)) { oldCatalogPerStream.get(streamDescriptor) .stream(newCatalogPerStream.get(streamDescriptor).getStream()); + changes.add(staticFormatDiff(transformation)); } } case ADD_STREAM -> { @@ -74,53 +123,27 @@ public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, // the catalog. streamAndConfigurationToAdd.getConfig() .selected(true); - configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd); + CatalogConverter.configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd.getStream(), + streamAndConfigurationToAdd.getConfig()); + CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfigurationToAdd, supportedDestinationSyncModes); } // TODO(mfsiega-airbyte): handle the case where the chosen sync mode isn't actually one of the // supported sync modes. oldCatalogPerStream.put(streamDescriptor, streamAndConfigurationToAdd); + changes.add(staticFormatDiff(transformation)); } } case REMOVE_STREAM -> { if (nonBreakingChangesPreference.equals(NonBreakingChangesPreference.PROPAGATE_FULLY)) { oldCatalogPerStream.remove(streamDescriptor); + changes.add(staticFormatDiff(transformation)); } } default -> throw new NotSupportedException("Not supported transformation."); } }); - return new AirbyteCatalog().streams(List.copyOf(oldCatalogPerStream.values())); - } - - private static void configureDefaultSyncModesForNewStream(final AirbyteStreamAndConfiguration streamToAdd) { - // TODO(mfsiega-airbyte): unite this with the default config generation in the CatalogConverter. - final var stream = streamToAdd.getStream(); - final var config = streamToAdd.getConfig(); - final boolean hasSourceDefinedCursor = stream.getSourceDefinedCursor() != null && stream.getSourceDefinedCursor(); - final boolean hasSourceDefinedPrimaryKey = stream.getSourceDefinedPrimaryKey() != null && !stream.getSourceDefinedPrimaryKey().isEmpty(); - final boolean supportsFullRefresh = stream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); - if (hasSourceDefinedCursor && hasSourceDefinedPrimaryKey) { // Source-defined cursor and primary key - config - .syncMode(SyncMode.INCREMENTAL) - .destinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .primaryKey(stream.getSourceDefinedPrimaryKey()); - } else if (hasSourceDefinedCursor && supportsFullRefresh) { // Source-defined cursor but no primary key. - // NOTE: we prefer Full Refresh | Overwrite to avoid the risk of an Incremental | Append sync - // blowing up their destination. - config - .syncMode(SyncMode.FULL_REFRESH) - .destinationSyncMode(DestinationSyncMode.OVERWRITE); - } else if (hasSourceDefinedCursor) { // Source-defined cursor but no primary key *and* no full-refresh supported. - // If *only* incremental is supported, we go with it. - config - .syncMode(SyncMode.INCREMENTAL) - .destinationSyncMode(DestinationSyncMode.APPEND); - } else { // No source-defined cursor at all. - config - .syncMode(SyncMode.FULL_REFRESH) - .destinationSyncMode(DestinationSyncMode.OVERWRITE); - } + return new UpdateSchemaResult(new AirbyteCatalog().streams(List.copyOf(oldCatalogPerStream.values())), changes); } @VisibleForTesting @@ -131,4 +154,20 @@ static Map extractStreamAndConf airbyteStreamAndConfiguration -> airbyteStreamAndConfiguration)); } + @VisibleForTesting + public static boolean containsBreakingChange(final CatalogDiff diff) { + for (final StreamTransform streamTransform : diff.getTransforms()) { + if (streamTransform.getTransformType() != StreamTransform.TransformTypeEnum.UPDATE_STREAM) { + continue; + } + + final boolean anyBreakingFieldTransforms = streamTransform.getUpdateStream().stream().anyMatch(FieldTransform::getBreaking); + if (anyBreakingFieldTransforms) { + return true; + } + } + + return false; + } + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java index d1b1d85be55..ab349611d08 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java @@ -220,6 +220,44 @@ public static io.airbyte.protocol.models.ConfiguredAirbyteCatalog toConfiguredPr .withStreams(streams); } + /** + * Set the default sync modes for an un-configured stream based on the stream properties. + *

+ * The logic is: - source-defined cursor and source-defined primary key -> INCREMENTAL, APPEND-DEDUP + * - source-defined cursor only or nothing defined by the source -> FULL REFRESH, OVERWRITE - + * source-defined cursor and full refresh not available as a sync method -> INCREMENTAL, APPEND + * + * @param streamToConfigure the stream for which we're picking a sync mode + * @param config the config to which we'll write the sync mode + */ + public static void configureDefaultSyncModesForNewStream(final AirbyteStream streamToConfigure, final AirbyteStreamConfiguration config) { + final boolean hasSourceDefinedCursor = streamToConfigure.getSourceDefinedCursor() != null && streamToConfigure.getSourceDefinedCursor(); + final boolean hasSourceDefinedPrimaryKey = + streamToConfigure.getSourceDefinedPrimaryKey() != null && !streamToConfigure.getSourceDefinedPrimaryKey().isEmpty(); + final boolean supportsFullRefresh = streamToConfigure.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); + if (hasSourceDefinedCursor && hasSourceDefinedPrimaryKey) { // Source-defined cursor and primary key + config + .syncMode(SyncMode.INCREMENTAL) + .destinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .primaryKey(streamToConfigure.getSourceDefinedPrimaryKey()); + } else if (hasSourceDefinedCursor && supportsFullRefresh) { // Source-defined cursor but no primary key. + // NOTE: we prefer Full Refresh | Overwrite to avoid the risk of an Incremental | Append sync + // blowing up their destination. + config + .syncMode(SyncMode.FULL_REFRESH) + .destinationSyncMode(DestinationSyncMode.OVERWRITE); + } else if (hasSourceDefinedCursor) { // Source-defined cursor but no primary key *and* no full-refresh supported. + // If *only* incremental is supported, we go with it. + config + .syncMode(SyncMode.INCREMENTAL) + .destinationSyncMode(DestinationSyncMode.APPEND); + } else { // No source-defined cursor at all. + config + .syncMode(SyncMode.FULL_REFRESH) + .destinationSyncMode(DestinationSyncMode.OVERWRITE); + } + } + @SuppressWarnings("LineLength") private static io.airbyte.api.model.generated.AirbyteStreamConfiguration generateDefaultConfiguration(final io.airbyte.api.model.generated.AirbyteStream stream, final Boolean suggestingStreams, @@ -239,11 +277,7 @@ private static io.airbyte.api.model.generated.AirbyteStreamConfiguration generat result.setSuggested(isSelected); result.setSelected(isSelected); - if (stream.getSupportedSyncModes().size() > 0) { - result.setSyncMode(stream.getSupportedSyncModes().get(0)); - } else { - result.setSyncMode(io.airbyte.api.model.generated.SyncMode.INCREMENTAL); - } + configureDefaultSyncModesForNewStream(stream, result); return result; } @@ -332,4 +366,40 @@ private static String streamDescriptorToStringForFieldSelection(final StreamDesc return String.format("%s/%s", streamDescriptor.getNamespace(), streamDescriptor.getName()); } + /** + * Ensure that the configured sync modes are compatible with the source and the destination. + *

+ * When we discover a new stream -- either during manual or auto schema refresh -- we want to pick + * some default sync modes. This depends both on the source-supported sync modes -- represented in + * the discovered catalog -- and the destination-supported sync modes. The latter is tricky because + * the place where we're generating the default configuration isn't associated with a particular + * destination. + *

+ * A longer-term fix would be to restructure how we generate this default config, but for now we use + * this to ensure that we've chosen defaults that work for the relevant sync. + * + * @param streamAndConfiguration the stream and configuration to check + * @param supportedDestinationSyncModes the sync modes supported by the destination + */ + public static void ensureCompatibleDestinationSyncMode(AirbyteStreamAndConfiguration streamAndConfiguration, + List supportedDestinationSyncModes) { + if (supportedDestinationSyncModes.contains(streamAndConfiguration.getConfig().getDestinationSyncMode())) { + return; + } + final var sourceSupportsFullRefresh = streamAndConfiguration.getStream().getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); + final var destinationSupportsOverwrite = supportedDestinationSyncModes.contains(DestinationSyncMode.OVERWRITE); + if (sourceSupportsFullRefresh && destinationSupportsOverwrite) { + // We prefer to fall back to Full Refresh | Overwrite if possible. + streamAndConfiguration.getConfig().syncMode(SyncMode.FULL_REFRESH).destinationSyncMode(DestinationSyncMode.OVERWRITE); + } else { + // If *that* isn't possible, we pick something that *is* supported. This isn't ideal, but we don't + // have a clean way + // to fail in this case today. + final var supportedSyncMode = streamAndConfiguration.getStream().getSupportedSyncModes().get(0); + final var supportedDestinationSyncMode = supportedDestinationSyncModes.get(0); + LOGGER.warn("Default sync modes are incompatible, so falling back to {} | {}", supportedSyncMode, supportedDestinationSyncMode); + streamAndConfiguration.getConfig().syncMode(supportedSyncMode).destinationSyncMode(supportedDestinationSyncMode); + } + } + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java deleted file mode 100644 index e12b5f35bb2..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import lombok.extern.slf4j.Slf4j; - -/** - * Default InstanceConfigurationHandler that returns the default "community" configuration for - * Airbyte. For Airbyte Pro, this singleton should be replaced with the - * {@link ProInstanceConfigurationHandler} that returns Pro-specific details. - */ -@Slf4j -@Singleton -public class DefaultInstanceConfigurationHandler implements InstanceConfigurationHandler { - - private final String webappUrl; - - // the injected webapp-url value defaults to `null` to preserve backwards compatibility. - // TODO remove the default value once configurations are standardized to always include airbyte.yml - public DefaultInstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final String webappUrl) { - this.webappUrl = webappUrl; - } - - @Override - public InstanceConfigurationResponse getInstanceConfiguration() { - return new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.COMMUNITY) - .licenseType(null) - .auth(null) - .webappUrl(webappUrl); - } - -} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java deleted file mode 100644 index 144fb16fc43..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; - -/** - * Handles requests for the Instance Configuration API endpoint. - */ -public interface InstanceConfigurationHandler { - - InstanceConfigurationResponse getInstanceConfiguration(); - -} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java deleted file mode 100644 index 06b21065909..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.AuthConfiguration; -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; -import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; -import io.airbyte.commons.license.ActiveAirbyteLicense; -import io.airbyte.commons.license.annotation.RequiresAirbyteProEnabled; -import io.micronaut.context.annotation.Replaces; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import lombok.extern.slf4j.Slf4j; - -/** - * Pro-specific version of the InstanceConfigurationHandler that includes license and auth - * configuration details for the instance. - */ -@Slf4j -@Singleton -@RequiresAirbyteProEnabled -@Replaces(DefaultInstanceConfigurationHandler.class) -public class ProInstanceConfigurationHandler implements InstanceConfigurationHandler { - - private final AirbyteKeycloakConfiguration keycloakConfiguration; - private final ActiveAirbyteLicense activeAirbyteLicense; - private final String webappUrl; - - public ProInstanceConfigurationHandler(final AirbyteKeycloakConfiguration keycloakConfiguration, - final ActiveAirbyteLicense activeAirbyteLicense, - @Value("${airbyte.webapp-url}") final String webappUrl) { - this.keycloakConfiguration = keycloakConfiguration; - this.activeAirbyteLicense = activeAirbyteLicense; - this.webappUrl = webappUrl; - } - - @Override - public InstanceConfigurationResponse getInstanceConfiguration() { - final LicenseTypeEnum licenseTypeEnum = LicenseTypeEnum.fromValue(activeAirbyteLicense.getLicenseType().getValue()); - - return new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.PRO) - .licenseType(licenseTypeEnum) - .auth(getAuthConfiguration()) - .webappUrl(webappUrl); - } - - private AuthConfiguration getAuthConfiguration() { - return new AuthConfiguration() - .clientId(getWebClientId()) - .defaultRealm(getAirbyteRealm()); - } - - private String getAirbyteRealm() { - return keycloakConfiguration.getAirbyteRealm(); - } - - private String getWebClientId() { - return keycloakConfiguration.getWebClientId(); - } - -} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java new file mode 100644 index 00000000000..5658b863e9c --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.micronaut.core.util.StringUtils; +import jakarta.inject.Singleton; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * Utility class that facilitates the extraction of values from HTTP request POST bodies. + */ +@Singleton +@Slf4j +public class AirbyteHttpRequestFieldExtractor { + + /** + * Extracts the requested ID from the HTTP request, if present. + * + * @param content The raw HTTP request as a string. + * @param idFieldName The name of the field/header that contains the ID. + * @return An {@link Optional} that may or may not contain the ID value extracted from the raw HTTP + * request. + */ + public Optional extractId(final String content, final String idFieldName) { + try { + final JsonNode json = Jsons.deserialize(content); + if (json != null) { + + final Optional idValue = extract(json, idFieldName); + + if (idValue.isEmpty()) { + log.debug("No match for field name '{}' in content '{}'.", idFieldName, content); + } else { + log.debug("Found '{}' for field '{}'", idValue, idFieldName); + return idValue; + } + } + } catch (final RuntimeException e) { + log.debug("Unable to extract ID field '{}' from content '{}'.", idFieldName, content, e); + } + + return Optional.empty(); + } + + private Optional extract(JsonNode jsonNode, String idFieldName) { + if (idFieldName.equals(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME)) { + log.debug("Try to extract list of ids for field {}", idFieldName); + return Optional.ofNullable(jsonNode.get(idFieldName)) + .map(Jsons::serialize) + .filter(StringUtils::hasText); + } else { + return Optional.ofNullable(jsonNode.get(idFieldName)) + .map(JsonNode::asText) + .filter(StringUtils::hasText); + } + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java new file mode 100644 index 00000000000..78b98ff7cfc --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.micronaut.context.annotation.Value; +import io.micronaut.context.event.BeanCreatedEvent; +import io.micronaut.context.event.BeanCreatedEventListener; +import io.micronaut.http.server.netty.NettyServerCustomizer; +import io.micronaut.http.server.netty.NettyServerCustomizer.Registry; +import io.netty.channel.Channel; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequestDecoder; +import jakarta.inject.Singleton; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom Netty customizer that registers the {@link AuthorizationServerHandler} with the Netty + * stream pipeline.
+ *
+ * This customizer registers the handler as the first in the pipeline to ensure that it can read + * and, if necessary, modify the incoming HTTP request to include a header that can be used to + * determine authorization. + */ +@Singleton +@Slf4j +public class AuthNettyServerCustomizer implements BeanCreatedEventListener { + + private final AuthorizationServerHandler authorizationServerHandler; + + private final Integer aggregatorMaxContentLength; + + public AuthNettyServerCustomizer(final AuthorizationServerHandler authorizationServerHandler, + @Value("${micronaut.server.netty.aggregator.max-content-length}") final Integer aggregatorMaxContentLength) { + this.authorizationServerHandler = authorizationServerHandler; + this.aggregatorMaxContentLength = aggregatorMaxContentLength; + } + + @Override + public Registry onCreated(final BeanCreatedEvent event) { + final NettyServerCustomizer.Registry registry = event.getBean(); + registry.register(new Customizer(null)); // + return registry; + } + + /** + * Custom {@link NettyServerCustomizer} that registers the {@link AuthorizationServerHandler} as the + * first handler in the Netty pipeline. + */ + private class Customizer implements NettyServerCustomizer { + + private final Channel channel; + + Customizer(final Channel channel) { + this.channel = channel; + } + + @Override + public NettyServerCustomizer specializeForChannel(final Channel channel, final ChannelRole role) { + return new Customizer(channel); + } + + @Override + public void onStreamPipelineBuilt() { + /* + * Register the handlers in reverse order so that the final order is: 1. Decoder 2. Aggregator 3. + * Authorization Handler + * + * This is to ensure that the full HTTP request with content is provided to the authorization + * handler. + */ + channel.pipeline() + .addFirst("authorizationServerHandler", authorizationServerHandler) + .addFirst("aggregator", new HttpObjectAggregator(aggregatorMaxContentLength)) + .addFirst("decoder", new HttpRequestDecoder()); + } + + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java new file mode 100644 index 00000000000..36b192cb057 --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +/** + * Collection of constants used to identify and extract ID values from a raw HTTP request for + * authentication purposes. + */ +public final class AuthenticationFields { + + /** + * Name of the field in HTTP request bodies that contains the connection ID value. + */ + public static final String CONNECTION_ID_FIELD_NAME = "connectionId"; + + /** + * Name of the field in HTTP request bodies that contains the destination ID value. + */ + public static final String DESTINATION_ID_FIELD_NAME = "destinationId"; + + /** + * Name of the field in HTTP request bodies that contains the job ID value. + */ + public static final String JOB_ID_FIELD_NAME = "id"; + + /** + * Alternative name of the field in HTTP request bodies that contains the job ID value. + */ + public static final String JOB_ID_ALT_FIELD_NAME = "jobId"; + + /** + * Name of the field in HTTP request bodies that contains the operation ID value. + */ + public static final String OPERATION_ID_FIELD_NAME = "operationId"; + + /** + * Name of the field in HTTP request bodies that contains the source ID value. + */ + public static final String SOURCE_ID_FIELD_NAME = "sourceId"; + + /** + * Name of the field in HTTP request bodies that contains the source definition ID value. + */ + public static final String SOURCE_DEFINITION_ID_FIELD_NAME = "sourceDefinitionId"; + + /** + * Name of the field in HTTP request bodies that contains the Airbyte-assigned user ID value. + */ + public static final String AIRBYTE_USER_ID_FIELD_NAME = "userId"; + + /** + * Name of the field in HTTP request bodies that contains the resource creator's Airbyte-assigned + * user ID value. + */ + public static final String CREATOR_USER_ID_FIELD_NAME = "creatorUserId"; + + /** + * Name of the field in HTTP request bodies that contains the external auth ID value. + */ + public static final String EXTERNAL_AUTH_ID_FIELD_NAME = "authUserId"; + + /** + * Name of the field in HTTP request bodies that contains the email value. + */ + public static final String EMAIL_FIELD_NAME = "email"; + + /** + * Name of the field in HTTP request bodies that contains the workspace ID value. + */ + public static final String WORKSPACE_ID_FIELD_NAME = "workspaceId"; + + /** + * Name of the field in HTTP request bodies that contains the workspace IDs value. + */ + public static final String WORKSPACE_IDS_FIELD_NAME = "workspaceIds"; + + /** + * Name of the field in HTTP request bodies that contains the config ID value - this is equivalent + * to the connection ID. + */ + public static final String CONFIG_ID_FIELD_NAME = "configId"; + + public static final String ORGANIZATION_ID_FIELD_NAME = "organizationId"; + + private AuthenticationFields() {} + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java new file mode 100644 index 00000000000..5a83c59672a --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.persistence.job.WorkspaceHelper; +import io.airbyte.validation.json.JsonValidationException; +import jakarta.inject.Singleton; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +/** + * Resolves organization or workspace IDs from HTTP headers. + */ +@Slf4j +@Singleton +public class AuthenticationHeaderResolver { + + private final WorkspaceHelper workspaceHelper; + + public AuthenticationHeaderResolver(final WorkspaceHelper workspaceHelper) { + this.workspaceHelper = workspaceHelper; + } + + /** + * Resolve corresponding organization ID. Currently we support two ways to resolve organization ID: + * 1. If the organization ID is provided in the header, we will use it directly. 2. Otherwise, we + * infer the workspace ID from the header and use the workspace ID to find the organization Id. + */ + public List resolveOrganization(final Map properties) { + log.debug("properties: {}", properties); + + if (properties.containsKey(ORGANIZATION_ID_HEADER)) { + return List.of(UUID.fromString(properties.get(ORGANIZATION_ID_HEADER))); + } + // Else, determine the organization from workspace related fields. + + final List workspaceIds = resolveWorkspace(properties); + if (workspaceIds == null) { + return null; + } + return workspaceIds.stream().map(workspaceId -> workspaceHelper.getOrganizationForWorkspace(workspaceId)).collect(Collectors.toList()); + } + + /** + * Resolves workspaces from header. + */ + @SuppressWarnings("PMD.CyclomaticComplexity") // This is an indication that the workspace ID as a group for auth needs refactoring + public List resolveWorkspace(final Map properties) { + log.debug("properties: {}", properties); + try { + if (properties.containsKey(WORKSPACE_ID_HEADER)) { + final String workspaceId = properties.get(WORKSPACE_ID_HEADER); + return List.of(UUID.fromString(workspaceId)); + } else if (properties.containsKey(CONNECTION_ID_HEADER)) { + final String connectionId = properties.get(CONNECTION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(connectionId))); + } else if (properties.containsKey(SOURCE_ID_HEADER) && properties.containsKey(DESTINATION_ID_HEADER)) { + final String destinationId = properties.get(DESTINATION_ID_HEADER); + final String sourceId = properties.get(SOURCE_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnection(UUID.fromString(sourceId), UUID.fromString(destinationId))); + } else if (properties.containsKey(DESTINATION_ID_HEADER)) { + final String destinationId = properties.get(DESTINATION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForDestinationId(UUID.fromString(destinationId))); + } else if (properties.containsKey(JOB_ID_HEADER)) { + final String jobId = properties.get(JOB_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForJobId(Long.valueOf(jobId))); + } else if (properties.containsKey(SOURCE_ID_HEADER)) { + final String sourceId = properties.get(SOURCE_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceId))); + } else if (properties.containsKey(SOURCE_DEFINITION_ID_HEADER)) { + final String sourceDefinitionId = properties.get(SOURCE_DEFINITION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceDefinitionId))); + } else if (properties.containsKey(OPERATION_ID_HEADER)) { + final String operationId = properties.get(OPERATION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForOperationId(UUID.fromString(operationId))); + } else if (properties.containsKey(CONFIG_ID_HEADER)) { + final String configId = properties.get(CONFIG_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(configId))); + } else if (properties.containsKey(WORKSPACE_IDS_HEADER)) { + return resolveWorkspaces(properties); + } else { + log.debug("Request does not contain any headers that resolve to a workspace ID."); + return null; + } + } catch (final JsonValidationException | ConfigNotFoundException e) { + log.debug("Unable to resolve workspace ID.", e); + return null; + } + } + + private List resolveWorkspaces(final Map properties) { + final String workspaceIds = properties.get(WORKSPACE_IDS_HEADER); + log.debug("workspaceIds from header: {}", workspaceIds); + if (workspaceIds != null) { + final List deserialized = Jsons.deserialize(workspaceIds, List.class); + return deserialized.stream().map(UUID::fromString).toList(); + } + log.debug("Request does not contain any headers that resolve to a list of workspace IDs."); + return null; + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java new file mode 100644 index 00000000000..6faecd212fd --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.micronaut.http.HttpHeaders; + +/** + * Collection of HTTP headers that are used to perform authentication/authorization. + */ +public final class AuthenticationHttpHeaders { + + /** + * Prefix that denotes an internal Airbyte header. + */ + public static final String AIRBYTE_HEADER_PREFIX = "X-Airbyte-"; + + /** + * Authorization header. + */ + public static final String AUTHORIZATION_HEADER = HttpHeaders.AUTHORIZATION; + + /** + * HTTP header that contains the connection ID for authorization purposes. + */ + public static final String CONNECTION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Connection-Id"; + + /** + * HTTP header that contains the destination ID for authorization purposes. + */ + public static final String DESTINATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Destination-Id"; + + /** + * HTTP header that contains the job ID for authorization purposes. + */ + public static final String JOB_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Job-Id"; + + /** + * HTTP header that contains the operation ID for authorization purposes. + */ + public static final String OPERATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Operation-Id"; + + /** + * HTTP header that contains the source ID for authorization purposes. + */ + public static final String SOURCE_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Source-Id"; + + /** + * HTTP header that contains the source definition ID for authorization purposes. + */ + public static final String SOURCE_DEFINITION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Source-Definition-Id"; + + /** + * HTTP header that contains the Airbyte-assigned user ID for authorization purposes. + */ + public static final String AIRBYTE_USER_ID_HEADER = AIRBYTE_HEADER_PREFIX + "User-Id"; + + /** + * HTTP header that contains the resource creator's Airbyte-assigned user ID for authorization + * purposes. + */ + public static final String CREATOR_USER_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Creator-User-Id"; + + /** + * HTTP header that contains the external auth ID for authorization purposes. + */ + public static final String EXTERNAL_AUTH_ID_HEADER = AIRBYTE_HEADER_PREFIX + "External-Auth-Id"; + + /** + * HTTP header that contains the auth user ID for authorization purposes. + */ + public static final String EMAIL_HEADER = AIRBYTE_HEADER_PREFIX + "Email"; + + /** + * HTTP header that contains the workspace ID for authorization purposes. + */ + public static final String WORKSPACE_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Workspace-Id"; + public static final String WORKSPACE_IDS_HEADER = AIRBYTE_HEADER_PREFIX + "Workspace-Ids"; + public static final String CONFIG_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Config-Id"; + + public static final String ORGANIZATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Organization-Id"; + + private AuthenticationHttpHeaders() {} + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java new file mode 100644 index 00000000000..97fd401f35c --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.AIRBYTE_USER_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CREATOR_USER_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.EMAIL_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.EXTERNAL_AUTH_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; + +/** + * Enumeration of the ID values that are used to perform authentication. These values are used to + * fetch roles associated with an authenticated user. + */ +public enum AuthenticationId { + + EXTERNAL_AUTH_ID(AuthenticationFields.EXTERNAL_AUTH_ID_FIELD_NAME, EXTERNAL_AUTH_ID_HEADER), + CONNECTION_ID(AuthenticationFields.CONNECTION_ID_FIELD_NAME, CONNECTION_ID_HEADER), + DESTINATION_ID_(AuthenticationFields.DESTINATION_ID_FIELD_NAME, DESTINATION_ID_HEADER), + EMAIL(AuthenticationFields.EMAIL_FIELD_NAME, EMAIL_HEADER), + JOB_ID(AuthenticationFields.JOB_ID_FIELD_NAME, JOB_ID_HEADER), + JOB_ID_ALT(AuthenticationFields.JOB_ID_ALT_FIELD_NAME, JOB_ID_HEADER), + OPERATION_ID(AuthenticationFields.OPERATION_ID_FIELD_NAME, OPERATION_ID_HEADER), + SOURCE_ID(AuthenticationFields.SOURCE_ID_FIELD_NAME, SOURCE_ID_HEADER), + SOURCE_DEFINITION_ID(AuthenticationFields.SOURCE_DEFINITION_ID_FIELD_NAME, SOURCE_DEFINITION_ID_HEADER), + AIRBYTE_USER_ID(AuthenticationFields.AIRBYTE_USER_ID_FIELD_NAME, AIRBYTE_USER_ID_HEADER), + CREATOR_USER_ID(AuthenticationFields.CREATOR_USER_ID_FIELD_NAME, CREATOR_USER_ID_HEADER), + WORKSPACE_ID(AuthenticationFields.WORKSPACE_ID_FIELD_NAME, WORKSPACE_ID_HEADER), + WORKSPACE_IDS(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME, WORKSPACE_IDS_HEADER), + CONFIG_ID(AuthenticationFields.CONFIG_ID_FIELD_NAME, CONFIG_ID_HEADER), + ORGANIZATION_ID(AuthenticationFields.ORGANIZATION_ID_FIELD_NAME, ORGANIZATION_ID_HEADER); + + private final String fieldName; + private final String httpHeader; + + AuthenticationId(final String fieldName, final String httpHeader) { + this.fieldName = fieldName; + this.httpHeader = httpHeader; + } + + public String getFieldName() { + return fieldName; + } + + public String getHttpHeader() { + return httpHeader; + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java new file mode 100644 index 00000000000..93a6231a825 --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpHeaders; +import jakarta.inject.Singleton; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom Netty {@link ChannelDuplexHandler} that intercepts all operations to ensure that headers + * required for authorization are populated prior to performing the security check. + */ +@Singleton +@Sharable +@Slf4j +public class AuthorizationServerHandler extends ChannelDuplexHandler { + + private final AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor; + + public AuthorizationServerHandler(final AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor) { + this.airbyteHttpRequestFieldExtractor = airbyteHttpRequestFieldExtractor; + } + + @Override + public void channelRead( + final ChannelHandlerContext context, + final Object message) { + + Object updatedMessage = message; + + if (FullHttpRequest.class.isInstance(message)) { + final FullHttpRequest fullHttpRequest = FullHttpRequest.class.cast(message); + updatedMessage = updateHeaders(fullHttpRequest); + } + + context.fireChannelRead(updatedMessage); + } + + /** + * Checks the payload of the raw HTTP request for ID fields that should be copied to an HTTP header + * in order to facilitate authorization via Micronaut Security. + * + * @param httpRequest The raw HTTP request as a {@link FullHttpRequest}. + * @return The potentially modified raw HTTP request as a {@link FullHttpRequest}. + */ + protected FullHttpRequest updateHeaders(final FullHttpRequest httpRequest) { + for (final AuthenticationId authenticationId : AuthenticationId.values()) { + final String contentAsString = StandardCharsets.UTF_8.decode(httpRequest.content().nioBuffer()).toString(); + log.debug("Checking HTTP request '{}' for field '{}'...", contentAsString, authenticationId.getFieldName()); + final Optional id = + airbyteHttpRequestFieldExtractor.extractId(contentAsString, authenticationId.getFieldName()); + if (id.isPresent()) { + log.debug("Found field '{}' with value '{}' in HTTP request body.", authenticationId.getFieldName(), id.get()); + addHeaderToRequest(authenticationId.getHttpHeader(), id.get(), httpRequest); + } else { + log.debug("Field '{}' not found in content.", authenticationId.getFieldName()); + } + } + + return httpRequest; + } + + /** + * Adds the provided header and value to the HTTP request represented by the {@link FullHttpRequest} + * if the header is not already present. + * + * @param headerName The name of the header. + * @param headerValue The value of the header. + * @param httpRequest The current HTTP request. + */ + protected void addHeaderToRequest(final String headerName, final Object headerValue, final FullHttpRequest httpRequest) { + final HttpHeaders httpHeaders = httpRequest.headers(); + if (!httpHeaders.contains(headerName)) { + log.debug("Adding HTTP header '{}' with value '{}' to request...", headerName, headerValue); + httpHeaders.add(headerName, headerValue.toString()); + } + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java index 32eec2e4326..4f288fd605d 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java @@ -20,6 +20,7 @@ import io.airbyte.commons.server.helpers.ConnectionHelpers; import io.airbyte.config.DataType; import io.airbyte.config.FieldSelectionData; +import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; import java.util.List; import org.junit.jupiter.api.Test; @@ -127,4 +128,48 @@ void testConvertToProtocolFieldSelection() throws JsonValidationException { assertEquals(ConnectionHelpers.generateBasicConfiguredAirbyteCatalog(), CatalogConverter.toConfiguredProtocol(catalog)); } + @Test + void testDiscoveredToApiDefaultSyncModesNoSourceCursor() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.FULL_REFRESH, actualSyncMode); + assertEquals(DestinationSyncMode.OVERWRITE, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorAndPrimaryKey() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true).withSourceDefinedPrimaryKey(List.of(List.of("unused"))); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.INCREMENTAL, actualSyncMode); + assertEquals(DestinationSyncMode.APPEND_DEDUP, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorNoPrimaryKey() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.FULL_REFRESH, actualSyncMode); + assertEquals(DestinationSyncMode.OVERWRITE, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorNoFullRefresh() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true) + .withSupportedSyncModes(List.of(io.airbyte.protocol.models.SyncMode.INCREMENTAL)); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.INCREMENTAL, actualSyncMode); + assertEquals(DestinationSyncMode.APPEND, actualDestinationSyncMode); + } + } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java index 695af43beab..40c18a58923 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java @@ -15,16 +15,20 @@ import io.airbyte.api.model.generated.ActorDefinitionVersionRead; import io.airbyte.api.model.generated.DestinationIdRequestBody; import io.airbyte.api.model.generated.SourceIdRequestBody; +import io.airbyte.commons.server.converters.ApiPojoConverters; import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; import io.airbyte.config.DestinationConnection; +import io.airbyte.config.NormalizationDestinationDefinitionConfig; import io.airbyte.config.ReleaseStage; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.validation.json.JsonValidationException; @@ -62,6 +66,7 @@ private ActorDefinitionVersion createActorDefinitionVersion() { return new ActorDefinitionVersion() .withActorDefinitionId(ACTOR_DEFINITION_ID) .withVersionId(UUID.randomUUID()) + .withSupportLevel(SupportLevel.NONE) .withReleaseStage(ReleaseStage.BETA) .withSupportState(SupportState.SUPPORTED) .withDockerRepository("airbyte/source-faker") @@ -69,9 +74,18 @@ private ActorDefinitionVersion createActorDefinitionVersion() { .withDocumentationUrl("https://docs.airbyte.io"); } + private ActorDefinitionVersion createActorDefinitionVersionWithNormalization() { + return createActorDefinitionVersion() + .withSupportsDbt(true) + .withNormalizationConfig(new NormalizationDestinationDefinitionConfig() + .withNormalizationRepository("repository") + .withNormalizationTag("dev") + .withNormalizationIntegrationType("integration-type")); + } + @ParameterizedTest @CsvSource({"true", "false"}) - void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault) + void testGetActorDefinitionVersionForSource(final boolean isOverrideApplied) throws JsonValidationException, ConfigNotFoundException, IOException { final UUID sourceId = UUID.randomUUID(); final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion(); @@ -79,30 +93,29 @@ void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault .withSourceId(sourceId) .withWorkspaceId(WORKSPACE_ID); - if (isSourceVersionDefault) { - sourceConnection.withDefaultVersionId(actorDefinitionVersion.getVersionId()); - } - when(mConfigRepository.getSourceConnection(sourceId)) .thenReturn(sourceConnection); when(mConfigRepository.getSourceDefinitionFromSource(sourceId)) .thenReturn(SOURCE_DEFINITION); - when(mActorDefinitionVersionHelper.getSourceVersion(SOURCE_DEFINITION, WORKSPACE_ID, sourceId)) - .thenReturn(actorDefinitionVersion); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(SOURCE_DEFINITION, WORKSPACE_ID, sourceId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(sourceId); final ActorDefinitionVersionRead actorDefinitionVersionRead = actorDefinitionVersionHandler.getActorDefinitionVersionForSourceId(sourceIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(isSourceVersionDefault) + .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) - .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)); assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getSourceConnection(sourceId); verify(mConfigRepository).getSourceDefinitionFromSource(sourceId); - verify(mActorDefinitionVersionHelper).getSourceVersion(SOURCE_DEFINITION, WORKSPACE_ID, sourceId); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(SOURCE_DEFINITION, WORKSPACE_ID, sourceId); verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); verifyNoMoreInteractions(mConfigRepository); verifyNoMoreInteractions(mActorDefinitionVersionHelper); @@ -110,7 +123,7 @@ void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault @ParameterizedTest @CsvSource({"true", "false"}) - void testGetActorDefinitionVersionForDestination(final boolean isDestinationVersionDefault) + void testGetActorDefinitionVersionForDestination(final boolean isOverrideApplied) throws JsonValidationException, ConfigNotFoundException, IOException { final UUID destinationId = UUID.randomUUID(); final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion(); @@ -118,30 +131,67 @@ void testGetActorDefinitionVersionForDestination(final boolean isDestinationVers .withDestinationId(destinationId) .withWorkspaceId(WORKSPACE_ID); - if (isDestinationVersionDefault) { - destinationConnection.withDefaultVersionId(actorDefinitionVersion.getVersionId()); - } + when(mConfigRepository.getDestinationConnection(destinationId)) + .thenReturn(destinationConnection); + when(mConfigRepository.getDestinationDefinitionFromDestination(destinationId)) + .thenReturn(DESTINATION_DEFINITION); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); + + final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId); + final ActorDefinitionVersionRead actorDefinitionVersionRead = + actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); + final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() + .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) + .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) + .dockerRepository(actorDefinitionVersion.getDockerRepository()) + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)); + + assertEquals(expectedRead, actorDefinitionVersionRead); + verify(mConfigRepository).getDestinationConnection(destinationId); + verify(mConfigRepository).getDestinationDefinitionFromDestination(destinationId); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); + verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @ParameterizedTest + @CsvSource({"true", "false"}) + void testGetActorDefinitionVersionForDestinationWithNormalization(final boolean isOverrideApplied) + throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID destinationId = UUID.randomUUID(); + final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersionWithNormalization(); + final DestinationConnection destinationConnection = new DestinationConnection() + .withDestinationId(destinationId) + .withWorkspaceId(WORKSPACE_ID); when(mConfigRepository.getDestinationConnection(destinationId)) .thenReturn(destinationConnection); when(mConfigRepository.getDestinationDefinitionFromDestination(destinationId)) .thenReturn(DESTINATION_DEFINITION); - when(mActorDefinitionVersionHelper.getDestinationVersion(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) - .thenReturn(actorDefinitionVersion); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId); final ActorDefinitionVersionRead actorDefinitionVersionRead = actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(isDestinationVersionDefault) + .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) - .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(actorDefinitionVersion.getSupportsDbt()) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(actorDefinitionVersion.getNormalizationConfig())); assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getDestinationConnection(destinationId); verify(mConfigRepository).getDestinationDefinitionFromDestination(destinationId); - verify(mActorDefinitionVersionHelper).getDestinationVersion(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); verifyNoMoreInteractions(mConfigRepository); verifyNoMoreInteractions(mActorDefinitionVersionHelper); @@ -166,14 +216,19 @@ void testCreateActorDefinitionVersionReadWithBreakingChange() throws IOException final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion().withSupportState(SupportState.DEPRECATED); when(mConfigRepository.listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion)).thenReturn(breakingChanges); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, false); final ActorDefinitionVersionRead actorDefinitionVersionRead = - actorDefinitionVersionHandler.createActorDefinitionVersionRead(actorDefinitionVersion, true); + actorDefinitionVersionHandler.createActorDefinitionVersionRead(versionWithOverrideStatus); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(true) + .isOverrideApplied(false) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.DEPRECATED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)) .breakingChanges(new ActorDefinitionVersionBreakingChanges() .minUpgradeDeadline(LocalDate.parse("2023-01-01")) .upcomingBreakingChanges(List.of( diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/AttemptHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/AttemptHandlerTest.java index 5d2310f8499..a7d8a1a0dd5 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/AttemptHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/AttemptHandlerTest.java @@ -21,6 +21,7 @@ import io.airbyte.api.model.generated.AttemptSyncConfig; import io.airbyte.api.model.generated.ConnectionState; import io.airbyte.api.model.generated.ConnectionStateType; +import io.airbyte.api.model.generated.CreateNewAttemptNumberResponse; import io.airbyte.api.model.generated.GlobalState; import io.airbyte.api.model.generated.LogRead; import io.airbyte.api.model.generated.SaveAttemptSyncConfigRequestBody; @@ -29,16 +30,21 @@ import io.airbyte.commons.server.converters.ApiPojoConverters; import io.airbyte.commons.server.converters.JobConverter; import io.airbyte.commons.server.errors.IdNotFoundKnownException; +import io.airbyte.commons.server.handlers.helpers.JobCreationAndStatusUpdateHelper; +import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.config.SyncStats; +import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptStatus; +import io.airbyte.persistence.job.models.Job; import java.io.IOException; import java.nio.file.Path; import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.UUID; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -48,6 +54,7 @@ class AttemptHandlerTest { JobConverter jobConverter; JobPersistence jobPersistence; + Path path; AttemptHandler handler; private static final UUID CONNECTION_ID = UUID.randomUUID(); @@ -60,7 +67,8 @@ class AttemptHandlerTest { public void init() { jobPersistence = Mockito.mock(JobPersistence.class); jobConverter = Mockito.mock(JobConverter.class); - handler = new AttemptHandler(jobPersistence, jobConverter); + path = Mockito.mock(Path.class); + handler = new AttemptHandler(jobPersistence, jobConverter, Mockito.mock(JobCreationAndStatusUpdateHelper.class), path); } @Test @@ -148,6 +156,29 @@ void testInternalHandlerSetsAttemptSyncConfig() throws Exception { assertEquals(expectedAttemptSyncConfig, attemptSyncConfigCapture.getValue()); } + @Test + void createAttemptNumber() throws IOException { + final int attemptNumber = 1; + final Job mJob = Mockito.mock(Job.class); + Mockito.when(mJob.getAttemptsCount()) + .thenReturn(ATTEMPT_NUMBER); + + Mockito.when(jobPersistence.getJob(JOB_ID)) + .thenReturn(mJob); + + Mockito.when(path.resolve(Mockito.anyString())) + .thenReturn(path); + + final Path expectedRoot = TemporalUtils.getJobRoot(path, String.valueOf(JOB_ID), ATTEMPT_NUMBER); + final Path expectedLogPath = expectedRoot.resolve(LogClientSingleton.LOG_FILENAME); + + Mockito.when(jobPersistence.createAttempt(JOB_ID, expectedLogPath)) + .thenReturn(attemptNumber); + + final CreateNewAttemptNumberResponse output = handler.createNewAttemptNumber(JOB_ID); + Assertions.assertThat(output.getAttemptNumber()).isEqualTo(attemptNumber); + } + @Test void getAttemptThrowsNotFound() throws Exception { when(jobPersistence.getAttemptForJob(anyLong(), anyInt())).thenReturn(Optional.empty()); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java index 9d554044d5c..85c169e3c03 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java @@ -40,6 +40,7 @@ import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -379,7 +380,7 @@ void whenPublishConnectorBuilderProjectThenCreateActorDefinition() throws IOExce .initialDeclarativeManifest(anyInitialManifest().manifest(A_MANIFEST).spec(A_SPEC))); verify(manifestInjector, times(1)).addInjectedDeclarativeManifest(A_SPEC); - verify(configRepository, times(1)).writeCustomSourceDefinitionAndDefaultVersion(eq(new StandardSourceDefinition() + verify(configRepository, times(1)).writeCustomConnectorMetadata(eq(new StandardSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_SOURCE_NAME) .withSourceType(SourceType.CUSTOM) @@ -391,6 +392,7 @@ void whenPublishConnectorBuilderProjectThenCreateActorDefinition() throws IOExce .withDockerRepository("airbyte/source-declarative-manifest") .withDockerImageTag(CDK_VERSION) .withSpec(adaptedConnectorSpecification) + .withSupportLevel(SupportLevel.NONE) .withReleaseStage(ReleaseStage.CUSTOM) .withDocumentationUrl(A_DOCUMENTATION_URL) .withProtocolVersion("0.2.0")), diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index 445fa7e5690..4f1c13bd03f 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -33,6 +34,7 @@ import io.airbyte.api.model.generated.PrivateDestinationDefinitionRead; import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.generated.ReleaseStage; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; @@ -52,12 +54,13 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; import io.airbyte.protocol.models.ConnectorSpecification; @@ -94,6 +97,7 @@ class DestinationDefinitionsHandlerTest { private ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private RemoteDefinitionsProvider remoteDefinitionsProvider; private DestinationHandler destinationHandler; + private SupportStateUpdater supportStateUpdater; private UUID workspaceId; private UUID organizationId; private FeatureFlagClient featureFlagClient; @@ -110,6 +114,7 @@ void setUp() { actorDefinitionHandlerHelper = mock(ActorDefinitionHandlerHelper.class); remoteDefinitionsProvider = mock(RemoteDefinitionsProvider.class); destinationHandler = mock(DestinationHandler.class); + supportStateUpdater = mock(SupportStateUpdater.class); workspaceId = UUID.randomUUID(); organizationId = UUID.randomUUID(); featureFlagClient = mock(TestClient.class); @@ -119,9 +124,8 @@ void setUp() { actorDefinitionHandlerHelper, remoteDefinitionsProvider, destinationHandler, + supportStateUpdater, featureFlagClient); - - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); } private StandardDestinationDefinition generateDestinationDefinition() { @@ -145,6 +149,7 @@ private ActorDefinitionVersion generateVersionFromDestinationDefinition(final St .withDocumentationUrl("https://hulu.com") .withSpec(spec) .withProtocolVersion("0.2.2") + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) @@ -165,6 +170,7 @@ private ActorDefinitionVersion generateCustomVersionFromDestinationDefinition(fi return generateVersionFromDestinationDefinition(destinationDefinition) .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) .withReleaseDate(null) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM) .withAllowedHosts(null); } @@ -196,6 +202,7 @@ void testListDestinations() throws IOException, URISyntaxException { .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -213,6 +220,7 @@ void testListDestinations() throws IOException, URISyntaxException { .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -251,6 +259,7 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -268,6 +277,7 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -336,6 +346,7 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -353,6 +364,7 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -396,6 +408,7 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -466,6 +479,7 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -505,6 +519,7 @@ void testGetDefinitionWithGrantForScope() throws JsonValidationException, Config .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -569,6 +584,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .destinationDefinitionId(newDestinationDefinition.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(newDestinationDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.CUSTOM) .supportsDbt(false) .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) @@ -583,7 +600,7 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newDestinationDefinition .withCustom(true) .withDefaultVersionId(null), @@ -632,6 +649,8 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio .destinationDefinitionId(newDestinationDefinition.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(newDestinationDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.CUSTOM) .supportsDbt(false) .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) @@ -647,7 +666,7 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreateForWorkspace.getWorkspaceId()); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newDestinationDefinition .withCustom(true) .withDefaultVersionId(null), @@ -672,7 +691,7 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), null); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion(newDestinationDefinition.withCustom(true).withDefaultVersionId(null), + verify(configRepository).writeCustomConnectorMetadata(newDestinationDefinition.withCustom(true).withDefaultVersionId(null), destinationDefinitionVersion, organizationId, ScopeType.ORGANIZATION); verifyNoMoreInteractions(actorDefinitionHandlerHelper); @@ -708,7 +727,7 @@ void testCreateCustomDestinationDefinitionShouldCheckProtocolVersion() throws UR verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository, never()).writeCustomDestinationDefinitionAndDefaultVersion(any(), any(), any(), any()); + verify(configRepository, never()).writeCustomConnectorMetadata(any(StandardDestinationDefinition.class), any(), any(), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -716,24 +735,26 @@ void testCreateCustomDestinationDefinitionShouldCheckProtocolVersion() throws UR @ParameterizedTest @ValueSource(booleans = {true, false}) @DisplayName("updateDestinationDefinition should correctly update a destinationDefinition") - void testUpdateDestination(final boolean ingestBreakingChangesFF) throws ConfigNotFoundException, IOException, JsonValidationException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(ingestBreakingChangesFF); + void testUpdateDestination(final boolean runSupportStateUpdaterFlagValue) throws ConfigNotFoundException, IOException, JsonValidationException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(runSupportStateUpdaterFlagValue); - when(configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId())).thenReturn(destinationDefinition); - when(configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId())) - .thenReturn(destinationDefinitionVersion); - final DestinationDefinitionRead currentDestination = destinationDefinitionsHandler - .getDestinationDefinition( - new DestinationDefinitionIdRequestBody().destinationDefinitionId(destinationDefinition.getDestinationDefinitionId())); - final String currentTag = currentDestination.getDockerImageTag(); final String newDockerImageTag = "averydifferenttag"; - assertNotEquals(newDockerImageTag, currentTag); - final StandardDestinationDefinition updatedDestination = Jsons.clone(destinationDefinition).withDefaultVersionId(null); final ActorDefinitionVersion updatedDestinationDefVersion = generateVersionFromDestinationDefinition(updatedDestination) - .withDockerImageTag(newDockerImageTag); + .withDockerImageTag(newDockerImageTag) + .withVersionId(UUID.randomUUID()); + + final StandardDestinationDefinition persistedUpdatedDestination = + Jsons.clone(updatedDestination).withDefaultVersionId(updatedDestinationDefVersion.getVersionId()); + + when(configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId())) + .thenReturn(destinationDefinition) // Call at the beginning of the method + .thenReturn(persistedUpdatedDestination); // Call after we've persisted + + when(configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId())) + .thenReturn(destinationDefinitionVersion); when(actorDefinitionHandlerHelper.defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom())).thenReturn(updatedDestinationDefVersion); @@ -750,12 +771,13 @@ void testUpdateDestination(final boolean ingestBreakingChangesFF) throws ConfigN verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedDestinationDefVersion, ActorType.DESTINATION); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion(updatedDestination, updatedDestinationDefVersion, breakingChanges); - if (ingestBreakingChangesFF) { - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeConnectorMetadata(updatedDestination, updatedDestinationDefVersion, breakingChanges); + if (runSupportStateUpdaterFlagValue) { + verify(supportStateUpdater).updateSupportStatesForDestinationDefinition(persistedUpdatedDestination); + } else { + verifyNoInteractions(supportStateUpdater); } - - verifyNoMoreInteractions(actorDefinitionHandlerHelper); + verifyNoMoreInteractions(actorDefinitionHandlerHelper, supportStateUpdater); } @Test @@ -783,7 +805,7 @@ void testOutOfProtocolRangeUpdateDestination() throws ConfigNotFoundException, I verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom()); - verify(configRepository, never()).writeDestinationDefinitionAndDefaultVersion(any(), any()); + verify(configRepository, never()).writeConnectorMetadata(any(StandardDestinationDefinition.class), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -825,6 +847,7 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -862,6 +885,7 @@ void testGrantDestinationDefinitionToOrganization() throws JsonValidationExcepti .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -919,6 +943,7 @@ void testCorrect() { Jsons.jsonNode(ImmutableMap.of("key", "val")))) .withTombstone(false) .withProtocolVersion("0.2.2") + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java new file mode 100644 index 00000000000..e9fec7a8116 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.api.model.generated.AuthConfiguration; +import io.airbyte.api.model.generated.InstanceConfigurationResponse; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.api.model.generated.WorkspaceUpdate; +import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.license.ActiveAirbyteLicense; +import io.airbyte.commons.license.AirbyteLicense; +import io.airbyte.commons.license.AirbyteLicense.LicenseType; +import io.airbyte.config.Configs.AirbyteEdition; +import io.airbyte.config.Organization; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.User; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class InstanceConfigurationHandlerTest { + + private static final String WEBAPP_URL = "http://localhost:8000"; + private static final String AIRBYTE_REALM = "airbyte"; + private static final String WEB_CLIENT_ID = "airbyte-webapp"; + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID USER_ID = UUID.randomUUID(); + private static final UUID ORGANIZATION_ID = UUID.randomUUID(); + private static final String DEFAULT_ORG_NAME = "Default Org Name"; + private static final String DEFAULT_USER_NAME = "Default User Name"; + + @Mock + private WorkspacePersistence mWorkspacePersistence; + @Mock + private UserPersistence mUserPersistence; + @Mock + private WorkspacesHandler mWorkspacesHandler; + @Mock + private OrganizationPersistence mOrganizationPersistence; + + private AirbyteKeycloakConfiguration keycloakConfiguration; + private ActiveAirbyteLicense activeAirbyteLicense; + private InstanceConfigurationHandler instanceConfigurationHandler; + + @BeforeEach + void setup() throws IOException { + keycloakConfiguration = new AirbyteKeycloakConfiguration(); + keycloakConfiguration.setAirbyteRealm(AIRBYTE_REALM); + keycloakConfiguration.setWebClientId(WEB_CLIENT_ID); + + activeAirbyteLicense = new ActiveAirbyteLicense(); + activeAirbyteLicense.setLicense(new AirbyteLicense(LicenseType.PRO)); + } + + @ParameterizedTest + @CsvSource({ + "true, true", + "true, false", + "false, true", + "false, false" + }) + void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSetupComplete) throws IOException { + stubGetDefaultUser(); + stubGetDefaultOrganization(); + + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)).thenReturn( + new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withInitialSetupComplete(isInitialSetupComplete)); + + instanceConfigurationHandler = getInstanceConfigurationHandler(isPro); + + final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() + .edition(isPro ? EditionEnum.PRO : EditionEnum.COMMUNITY) + .webappUrl(WEBAPP_URL) + .licenseType(isPro ? LicenseTypeEnum.PRO : null) + .auth(isPro ? new AuthConfiguration() + .clientId(WEB_CLIENT_ID) + .defaultRealm(AIRBYTE_REALM) : null) + .initialSetupComplete(isInitialSetupComplete) + .defaultUserId(USER_ID) + .defaultOrganizationId(ORGANIZATION_ID) + .defaultWorkspaceId(WORKSPACE_ID); + + final InstanceConfigurationResponse actual = instanceConfigurationHandler.getInstanceConfiguration(); + + assertEquals(expected, actual); + } + + @Test + void testSetupInstanceConfigurationAlreadySetup() throws IOException { + stubGetDefaultOrganization(); + + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)).thenReturn( + new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withInitialSetupComplete(true)); // already setup, should trigger an error + + instanceConfigurationHandler = getInstanceConfigurationHandler(true); + + assertThrows(IllegalStateException.class, () -> instanceConfigurationHandler.setupInstanceConfiguration( + new InstanceConfigurationSetupRequestBody() + .email("test@mail.com") + .displaySetupWizard(false) + .initialSetupComplete(false) + .anonymousDataCollection(false))); + } + + @ParameterizedTest + @CsvSource({ + "true, true", + "true, false", + "false, true", + "false, false" + }) + void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean orgNamePresent) + throws IOException, JsonValidationException, ConfigNotFoundException { + + stubGetDefaultOrganization(); + stubGetDefaultUser(); + + // first time default workspace is fetched, initial setup complete is false. + // second time is after the workspace is updated, so initial setup complete is true. + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)) + .thenReturn( + new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withInitialSetupComplete(false)) + .thenReturn( + new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withInitialSetupComplete(true)); + + instanceConfigurationHandler = getInstanceConfigurationHandler(true); + + final String email = "test@airbyte.com"; + + final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() + .edition(EditionEnum.PRO) + .webappUrl(WEBAPP_URL) + .licenseType(LicenseTypeEnum.PRO) + .auth(new AuthConfiguration() + .clientId(WEB_CLIENT_ID) + .defaultRealm(AIRBYTE_REALM)) + .initialSetupComplete(true) + .defaultUserId(USER_ID) + .defaultOrganizationId(ORGANIZATION_ID) + .defaultWorkspaceId(WORKSPACE_ID); + + final InstanceConfigurationSetupRequestBody requestBody = new InstanceConfigurationSetupRequestBody() + .email(email) + .displaySetupWizard(true) + .anonymousDataCollection(true) + .initialSetupComplete(true); + + String expectedUserName = DEFAULT_USER_NAME; + if (userNamePresent) { + expectedUserName = "test user"; + requestBody.setUserName(expectedUserName); + } + + String expectedOrgName = DEFAULT_ORG_NAME; + if (orgNamePresent) { + expectedOrgName = "test org"; + requestBody.setOrganizationName(expectedOrgName); + } + + final InstanceConfigurationResponse actual = instanceConfigurationHandler.setupInstanceConfiguration(requestBody); + + assertEquals(expected, actual); + + // verify the user was updated with the email and name from the request + verify(mUserPersistence).writeUser(eq(new User() + .withUserId(USER_ID) + .withEmail(email) + .withName(expectedUserName))); + + // verify the organization was updated with the name from the request + verify(mOrganizationPersistence).updateOrganization(eq(new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName(expectedOrgName) + .withEmail(email) + .withUserId(USER_ID))); + + verify(mWorkspacesHandler).updateWorkspace(eq(new WorkspaceUpdate() + .workspaceId(WORKSPACE_ID) + .email(email) + .displaySetupWizard(true) + .anonymousDataCollection(true) + .initialSetupComplete(true))); + } + + private void stubGetDefaultUser() throws IOException { + when(mUserPersistence.getDefaultUser()).thenReturn( + Optional.of(new User() + .withUserId(USER_ID) + .withName(DEFAULT_USER_NAME))); + } + + private void stubGetDefaultOrganization() throws IOException { + when(mOrganizationPersistence.getDefaultOrganization()).thenReturn( + Optional.of(new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName(DEFAULT_ORG_NAME) + .withUserId(USER_ID))); + } + + private InstanceConfigurationHandler getInstanceConfigurationHandler(final boolean isPro) { + return new InstanceConfigurationHandler( + WEBAPP_URL, + isPro ? AirbyteEdition.PRO : AirbyteEdition.COMMUNITY, + isPro ? Optional.of(keycloakConfiguration) : Optional.empty(), + isPro ? Optional.of(activeAirbyteLicense) : Optional.empty(), + mWorkspacePersistence, + mWorkspacesHandler, + mUserPersistence, + mOrganizationPersistence); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index 95a753e4eca..3fa1af6c9e0 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -38,7 +38,6 @@ import io.airbyte.api.model.generated.ConnectionStreamRequestBody; import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationCoreConfig; -import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; import io.airbyte.api.model.generated.DestinationIdRequestBody; import io.airbyte.api.model.generated.DestinationUpdate; @@ -56,8 +55,6 @@ import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.SourceAutoPropagateChange; import io.airbyte.api.model.generated.SourceCoreConfig; -import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; -import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; import io.airbyte.api.model.generated.SourceDiscoverSchemaRead; import io.airbyte.api.model.generated.SourceDiscoverSchemaRequestBody; import io.airbyte.api.model.generated.SourceIdRequestBody; @@ -92,12 +89,15 @@ import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobTypeResourceLimit; import io.airbyte.config.JobTypeResourceLimit.JobType; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.OperatorWebhook; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -120,6 +120,7 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.StreamDescriptor; @@ -214,6 +215,23 @@ class SchedulerHandlerTest { public static final String A_DIFFERENT_STREAM = "aDifferentStream"; private static final ResourceRequirements RESOURCE_REQUIREMENT = new ResourceRequirements().withCpuLimit("1.0").withCpuRequest("0.5"); + private static final StandardDestinationDefinition SOME_DESTINATION_DEFINITION = new StandardDestinationDefinition() + .withDestinationDefinitionId(UUID.randomUUID()); + private static final ActorDefinitionVersion SOME_ACTOR_DEFINITION = new ActorDefinitionVersion().withSpec( + new ConnectorSpecification() + .withSupportedDestinationSyncModes(List.of(DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND, DestinationSyncMode.APPEND_DEDUP)) + .withDocumentationUrl(URI.create("unused"))); + + private static final StandardSyncOperation NORMALIZATION_OPERATION = new StandardSyncOperation() + .withOperatorType(StandardSyncOperation.OperatorType.NORMALIZATION) + .withOperatorNormalization(new OperatorNormalization()); + + private static final UUID NORMALIZATION_OPERATION_ID = UUID.randomUUID(); + + public static final StandardSyncOperation WEBHOOK_OPERATION = new StandardSyncOperation() + .withOperatorType(StandardSyncOperation.OperatorType.WEBHOOK) + .withOperatorWebhook(new OperatorWebhook()); + private static final UUID WEBHOOK_OPERATION_ID = UUID.randomUUID(); private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; @@ -237,6 +255,7 @@ class SchedulerHandlerTest { private SyncJobFactory jobFactory; private JobNotifier jobNotifier; private JobTracker jobTracker; + private ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler; @BeforeEach void setup() throws JsonValidationException, ConfigNotFoundException, IOException { @@ -257,6 +276,8 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio synchronousSchedulerClient = mock(SynchronousSchedulerClient.class); configRepository = mock(ConfigRepository.class); when(configRepository.getStandardSync(any())).thenReturn(new StandardSync().withStatus(StandardSync.Status.ACTIVE)); + when(configRepository.getStandardDestinationDefinition(any())).thenReturn(SOME_DESTINATION_DEFINITION); + when(configRepository.getDestinationDefinitionFromConnection(any())).thenReturn(SOME_DESTINATION_DEFINITION); secretsRepositoryWriter = mock(SecretsRepositoryWriter.class); jobPersistence = mock(JobPersistence.class); eventRunner = mock(EventRunner.class); @@ -264,17 +285,23 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); webUrlHelper = mock(WebUrlHelper.class); actorDefinitionVersionHelper = mock(ActorDefinitionVersionHelper.class); + when(actorDefinitionVersionHelper.getDestinationVersion(any(), any())).thenReturn(SOME_ACTOR_DEFINITION); streamResetPersistence = mock(StreamResetPersistence.class); oAuthConfigSupplier = mock(OAuthConfigSupplier.class); jobCreator = mock(JobCreator.class); jobFactory = mock(SyncJobFactory.class); jobNotifier = mock(JobNotifier.class); jobTracker = mock(JobTracker.class); + connectorDefinitionSpecificationHandler = mock(ConnectorDefinitionSpecificationHandler.class); jobConverter = spy(new JobConverter(WorkerEnvironment.DOCKER, LogConfigs.EMPTY)); featureFlagClient = mock(TestClient.class); + when(connectorDefinitionSpecificationHandler.getDestinationSpecification(any())).thenReturn(new DestinationDefinitionSpecificationRead() + .supportedDestinationSyncModes( + List.of(io.airbyte.api.model.generated.DestinationSyncMode.OVERWRITE, io.airbyte.api.model.generated.DestinationSyncMode.APPEND))); + schedulerHandler = new SchedulerHandler( configRepository, secretsRepositoryWriter, @@ -289,7 +316,7 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio webUrlHelper, actorDefinitionVersionHelper, featureFlagClient, - streamResetPersistence, oAuthConfigSupplier, jobCreator, jobFactory, jobNotifier, jobTracker); + streamResetPersistence, oAuthConfigSupplier, jobCreator, jobFactory, jobNotifier, jobTracker, connectorDefinitionSpecificationHandler); } @Test @@ -312,7 +339,10 @@ void createJob() throws JsonValidationException, ConfigNotFoundException, IOExce @Test @DisplayName("Test reset job creation") void createResetJob() throws JsonValidationException, ConfigNotFoundException, IOException { - final StandardSync standardSync = new StandardSync().withDestinationId(DESTINATION_ID); + Mockito.when(configRepository.getStandardSyncOperation(NORMALIZATION_OPERATION_ID)).thenReturn(NORMALIZATION_OPERATION); + Mockito.when(configRepository.getStandardSyncOperation(WEBHOOK_OPERATION_ID)).thenReturn(WEBHOOK_OPERATION); + final StandardSync standardSync = + new StandardSync().withDestinationId(DESTINATION_ID).withOperationIds(List.of(NORMALIZATION_OPERATION_ID, WEBHOOK_OPERATION_ID)); Mockito.when(configRepository.getStandardSync(CONNECTION_ID)).thenReturn(standardSync); final DestinationConnection destination = new DestinationConnection() .withDestinationId(DESTINATION_ID) @@ -334,7 +364,7 @@ void createResetJob() throws JsonValidationException, ConfigNotFoundException, I Mockito .when(jobCreator.createResetConnectionJob(destination, standardSync, destinationDefinition, actorDefinitionVersion, DOCKER_IMAGE_NAME, destinationVersion, - false, List.of(), + false, List.of(NORMALIZATION_OPERATION), streamsToReset, WORKSPACE_ID)) .thenReturn(Optional.of(JOB_ID)); @@ -449,137 +479,6 @@ void testCheckSourceConnectionFromUpdate() throws IOException, JsonValidationExc verify(actorDefinitionVersionHelper, times(2)).getSourceVersion(sourceDefinition, source.getWorkspaceId(), source.getSourceId()); } - @Test - void testGetSourceSpec() throws JsonValidationException, IOException, ConfigNotFoundException { - final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId = - new SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); - - final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName(NAME) - .withSourceDefinitionId(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); - when(configRepository.getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId())) - .thenReturn(sourceDefinition); - when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId())) - .thenReturn(new ActorDefinitionVersion() - .withDockerImageTag(SOURCE_DOCKER_TAG) - .withSpec(CONNECTOR_SPECIFICATION)); - - final SourceDefinitionSpecificationRead response = schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId); - - verify(configRepository).getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); - verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); - assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); - } - - @Test - void testGetSourceSpecForSourceId() throws JsonValidationException, IOException, ConfigNotFoundException { - final UUID sourceId = UUID.randomUUID(); - final UUID workspaceId = UUID.randomUUID(); - final UUID sourceDefinitionId = UUID.randomUUID(); - - final SourceIdRequestBody sourceIdRequestBody = - new SourceIdRequestBody() - .sourceId(sourceId); - - final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName(NAME) - .withSourceDefinitionId(sourceDefinitionId); - when(configRepository.getSourceConnection(sourceId)).thenReturn( - new SourceConnection() - .withSourceId(sourceId) - .withWorkspaceId(workspaceId) - .withSourceDefinitionId(sourceDefinitionId)); - when(configRepository.getStandardSourceDefinition(sourceDefinitionId)) - .thenReturn(sourceDefinition); - when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, workspaceId, sourceId)) - .thenReturn(new ActorDefinitionVersion() - .withDockerRepository(SOURCE_DOCKER_REPO) - .withDockerImageTag(SOURCE_DOCKER_TAG) - .withSpec(CONNECTOR_SPECIFICATION)); - - final SourceDefinitionSpecificationRead response = schedulerHandler.getSpecificationForSourceId(sourceIdRequestBody); - - verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, workspaceId, sourceId); - assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); - } - - @Test - void testGetSourceSpecWithoutDocs() throws JsonValidationException, IOException, ConfigNotFoundException { - final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId = - new SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); - - final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() - .withName(NAME) - .withSourceDefinitionId(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); - when(configRepository.getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId())) - .thenReturn(sourceDefinition); - when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId())) - .thenReturn(new ActorDefinitionVersion() - .withDockerRepository(SOURCE_DOCKER_REPO) - .withDockerImageTag(SOURCE_DOCKER_TAG) - .withSpec(CONNECTOR_SPECIFICATION_WITHOUT_DOCS_URL)); - - final SourceDefinitionSpecificationRead response = schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId); - - verify(configRepository).getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); - verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); - assertEquals(CONNECTOR_SPECIFICATION_WITHOUT_DOCS_URL.getConnectionSpecification(), response.getConnectionSpecification()); - } - - @Test - void testGetDestinationSpec() throws JsonValidationException, IOException, ConfigNotFoundException { - final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId = - new DestinationDefinitionIdWithWorkspaceId().destinationDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); - - final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName(NAME) - .withDestinationDefinitionId(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId()); - when(configRepository.getStandardDestinationDefinition(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId())) - .thenReturn(destinationDefinition); - when(actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, - destinationDefinitionIdWithWorkspaceId.getWorkspaceId())) - .thenReturn(new ActorDefinitionVersion() - .withDockerImageTag(DESTINATION_DOCKER_TAG) - .withSpec(CONNECTOR_SPECIFICATION)); - - final DestinationDefinitionSpecificationRead response = schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId); - - verify(configRepository).getStandardDestinationDefinition(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId()); - verify(actorDefinitionVersionHelper).getDestinationVersion(destinationDefinition, - destinationDefinitionIdWithWorkspaceId.getWorkspaceId()); - assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); - } - - @Test - void testGetDestinationSpecForDestinationId() throws JsonValidationException, IOException, ConfigNotFoundException { - final UUID destinationId = UUID.randomUUID(); - final UUID workspaceId = UUID.randomUUID(); - final UUID destinationDefinitionId = UUID.randomUUID(); - final DestinationIdRequestBody destinationIdRequestBody = - new DestinationIdRequestBody() - .destinationId(destinationId); - - final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() - .withName(NAME) - .withDestinationDefinitionId(destinationDefinitionId); - when(configRepository.getDestinationConnection(destinationId)).thenReturn( - new DestinationConnection() - .withDestinationId(destinationId) - .withWorkspaceId(workspaceId) - .withDestinationDefinitionId(destinationDefinitionId)); - when(configRepository.getStandardDestinationDefinition(destinationDefinitionId)) - .thenReturn(destinationDefinition); - when(actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, workspaceId, destinationId)) - .thenReturn(new ActorDefinitionVersion() - .withDockerImageTag(DESTINATION_DOCKER_TAG) - .withSpec(CONNECTOR_SPECIFICATION)); - - final DestinationDefinitionSpecificationRead response = schedulerHandler.getSpecificationForDestinationId(destinationIdRequestBody); - - verify(actorDefinitionVersionHelper).getDestinationVersion(destinationDefinition, workspaceId, destinationId); - assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); - } - @Test void testCheckDestinationConnectionFromDestinationId() throws IOException, JsonValidationException, ConfigNotFoundException { final DestinationConnection destination = DestinationHelpers.generateDestination(UUID.randomUUID()); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index 71495ed40d8..824433cd921 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -34,6 +35,7 @@ import io.airbyte.api.model.generated.SourceDefinitionUpdate; import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.SourceReadList; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; @@ -53,11 +55,12 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; @@ -92,6 +95,7 @@ class SourceDefinitionsHandlerTest { private ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private RemoteDefinitionsProvider remoteDefinitionsProvider; private SourceHandler sourceHandler; + private SupportStateUpdater supportStateUpdater; private UUID workspaceId; private UUID organizationId; private FeatureFlagClient featureFlagClient; @@ -104,6 +108,7 @@ void setUp() { actorDefinitionHandlerHelper = mock(ActorDefinitionHandlerHelper.class); remoteDefinitionsProvider = mock(RemoteDefinitionsProvider.class); sourceHandler = mock(SourceHandler.class); + supportStateUpdater = mock(SupportStateUpdater.class); workspaceId = UUID.randomUUID(); organizationId = UUID.randomUUID(); sourceDefinition = generateSourceDefinition(); @@ -111,7 +116,7 @@ void setUp() { featureFlagClient = mock(TestClient.class); sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, uuidSupplier, actorDefinitionHandlerHelper, remoteDefinitionsProvider, sourceHandler, - featureFlagClient); + supportStateUpdater, featureFlagClient); } private StandardSourceDefinition generateSourceDefinition() { @@ -136,6 +141,7 @@ private ActorDefinitionVersion generateVersionFromSourceDefinition(final Standar .withDockerRepository("dockerstuff") .withDockerImageTag("12.3") .withSpec(spec) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withAllowedHosts(new AllowedHosts().withHosts(List.of("host1", "host2"))) @@ -156,6 +162,7 @@ private ActorDefinitionVersion generateCustomVersionFromSourceDefinition(final S return generateVersionFromSourceDefinition(sourceDefinition) .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) .withReleaseDate(null) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM) .withAllowedHosts(null); } @@ -177,6 +184,7 @@ void testListSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -191,6 +199,7 @@ void testListSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -224,6 +233,7 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -238,6 +248,7 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -299,6 +310,7 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -313,6 +325,7 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -349,6 +362,7 @@ void testGetSourceDefinition() throws JsonValidationException, ConfigNotFoundExc .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -411,6 +425,7 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -444,6 +459,7 @@ void testGetDefinitionWithGrantForScope() throws JsonValidationException, Config .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -506,6 +522,8 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException { .sourceDefinitionId(newSourceDefinition.getSourceDefinitionId()) .icon(SourceDefinitionsHandler.loadIcon(newSourceDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.COMMUNITY) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() @@ -518,7 +536,7 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException { verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newSourceDefinition .withCustom(true) .withDefaultVersionId(null), @@ -567,6 +585,8 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO .sourceDefinitionId(newSourceDefinition.getSourceDefinitionId()) .icon(SourceDefinitionsHandler.loadIcon(newSourceDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.COMMUNITY) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() @@ -580,7 +600,7 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreateForWorkspace.getWorkspaceId()); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newSourceDefinition .withCustom(true) .withDefaultVersionId(null), @@ -605,7 +625,7 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), null); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion(newSourceDefinition.withCustom(true).withDefaultVersionId(null), + verify(configRepository).writeCustomConnectorMetadata(newSourceDefinition.withCustom(true).withDefaultVersionId(null), sourceDefinitionVersion, organizationId, ScopeType.ORGANIZATION); verifyNoMoreInteractions(actorDefinitionHandlerHelper); @@ -641,7 +661,7 @@ void testCreateCustomSourceDefinitionShouldCheckProtocolVersion() throws URISynt verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository, never()).writeCustomSourceDefinitionAndDefaultVersion(any(), any(), any(), any()); + verify(configRepository, never()).writeCustomConnectorMetadata(any(StandardSourceDefinition.class), any(), any(), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -649,24 +669,26 @@ void testCreateCustomSourceDefinitionShouldCheckProtocolVersion() throws URISynt @ParameterizedTest @ValueSource(booleans = {true, false}) @DisplayName("updateSourceDefinition should correctly update a sourceDefinition") - void testUpdateSource(final boolean ingestBreakingChangesFF) throws ConfigNotFoundException, IOException, JsonValidationException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(ingestBreakingChangesFF); + void testUpdateSource(final boolean runSupportStateUpdaterFlagValue) throws ConfigNotFoundException, IOException, JsonValidationException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(runSupportStateUpdaterFlagValue); - when(configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())).thenReturn(sourceDefinition); - when(configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId())) - .thenReturn(sourceDefinitionVersion); - final SourceDefinitionRead currentSource = sourceDefinitionsHandler - .getSourceDefinition( - new SourceDefinitionIdRequestBody().sourceDefinitionId(sourceDefinition.getSourceDefinitionId())); - final String currentTag = currentSource.getDockerImageTag(); final String newDockerImageTag = "averydifferenttag"; - assertNotEquals(newDockerImageTag, currentTag); - final StandardSourceDefinition updatedSource = Jsons.clone(sourceDefinition).withDefaultVersionId(null); final ActorDefinitionVersion updatedSourceDefVersion = generateVersionFromSourceDefinition(updatedSource) - .withDockerImageTag(newDockerImageTag); + .withDockerImageTag(newDockerImageTag) + .withVersionId(UUID.randomUUID()); + + final StandardSourceDefinition persistedUpdatedSource = + Jsons.clone(updatedSource).withDefaultVersionId(updatedSourceDefVersion.getVersionId()); + + when(configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())) + .thenReturn(sourceDefinition) // Call at the beginning of the method + .thenReturn(persistedUpdatedSource); // Call after we've persisted + + when(configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId())) + .thenReturn(sourceDefinitionVersion); when(actorDefinitionHandlerHelper.defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom())).thenReturn(updatedSourceDefVersion); @@ -683,12 +705,13 @@ void testUpdateSource(final boolean ingestBreakingChangesFF) throws ConfigNotFou verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedSourceDefVersion, ActorType.SOURCE); - verify(configRepository).writeSourceDefinitionAndDefaultVersion(updatedSource, updatedSourceDefVersion, breakingChanges); - if (ingestBreakingChangesFF) { - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeConnectorMetadata(updatedSource, updatedSourceDefVersion, breakingChanges); + if (runSupportStateUpdaterFlagValue) { + verify(supportStateUpdater).updateSupportStatesForSourceDefinition(persistedUpdatedSource); + } else { + verifyNoInteractions(supportStateUpdater); } - - verifyNoMoreInteractions(actorDefinitionHandlerHelper); + verifyNoMoreInteractions(actorDefinitionHandlerHelper, supportStateUpdater); } @Test @@ -716,7 +739,7 @@ void testOutOfProtocolRangeUpdateSource() throws ConfigNotFoundException, IOExce verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom()); - verify(configRepository, never()).writeSourceDefinitionAndDefaultVersion(any(), any()); + verify(configRepository, never()).writeConnectorMetadata(any(StandardSourceDefinition.class), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -757,6 +780,7 @@ void testGrantSourceDefinitionToWorkspace() throws JsonValidationException, Conf .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -791,6 +815,7 @@ void testGrantSourceDefinitionToOrganization() throws JsonValidationException, C .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -824,6 +849,32 @@ void testRevokeSourceDefinition() throws IOException { verify(configRepository).deleteActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), organizationId, ScopeType.ORGANIZATION); } + @Test + @DisplayName("should transform support level none to none") + void testNoneSupportLevel() { + final ConnectorRegistrySourceDefinition registrySourceDefinition = new ConnectorRegistrySourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withName("some-source") + .withDocumentationUrl("https://airbyte.com") + .withDockerRepository("dockerrepo") + .withDockerImageTag("1.2.4") + .withIcon("source.svg") + .withSpec(new ConnectorSpecification().withConnectionSpecification( + Jsons.jsonNode(ImmutableMap.of("key", "val")))) + .withTombstone(false) + .withSupportLevel(io.airbyte.config.SupportLevel.NONE) + .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) + .withReleaseDate(TODAY_DATE_STRING) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); + when(remoteDefinitionsProvider.getSourceDefinitions()).thenReturn(Collections.singletonList(registrySourceDefinition)); + + final SourceDefinitionRead expectedRead = + SourceDefinitionsHandler.buildSourceDefinitionRead(ConnectorRegistryConverters.toStandardSourceDefinition(registrySourceDefinition), + ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)); + + assertEquals(expectedRead.getSupportLevel(), SupportLevel.NONE); + } + @SuppressWarnings("TypeName") @Nested @DisplayName("listLatest") @@ -842,6 +893,7 @@ void testCorrect() { .withSpec(new ConnectorSpecification().withConnectionSpecification( Jsons.jsonNode(ImmutableMap.of("key", "val")))) .withTombstone(false) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); @@ -851,10 +903,11 @@ void testCorrect() { assertEquals(1, sourceDefinitionReadList.size()); final var sourceDefinitionRead = sourceDefinitionReadList.get(0); - assertEquals( + final SourceDefinitionRead expectedRead = SourceDefinitionsHandler.buildSourceDefinitionRead(ConnectorRegistryConverters.toStandardSourceDefinition(registrySourceDefinition), - ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)), - sourceDefinitionRead); + ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)); + + assertEquals(expectedRead, sourceDefinitionRead); } @Test diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java index 43f02097cad..eba9fda6c96 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java @@ -184,6 +184,10 @@ private io.airbyte.api.model.generated.NotificationSettings generateApiNotificat .sendOnSyncDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) .sendOnSyncDisabledWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)); } @@ -199,6 +203,10 @@ private io.airbyte.api.model.generated.NotificationSettings generateDefaultApiNo .sendOnSyncDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) .sendOnSyncDisabledWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)); } @@ -732,11 +740,11 @@ void testListWorkspacesInOrgNoKeyword() throws Exception { new ListWorkspacesInOrganizationRequestBody().organizationId(ORGANIZAION_ID).pagination(new Pagination().pageSize(100).rowOffset(0)); List expectedWorkspaces = List.of(generateWorkspace(), generateWorkspace()); - when(workspacePersistence.listWorkspacesByOrganizationId(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), + when(workspacePersistence.listWorkspacesByOrganizationIdPaginated(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), Optional.empty())) .thenReturn(expectedWorkspaces); WorkspaceReadList result = workspacesHandler.listWorkspacesInOrganization(request); - assertEquals(result.getWorkspaces().size(), 2); + assertEquals(2, result.getWorkspaces().size()); } @Test @@ -745,11 +753,11 @@ void testListWorkspacesInOrgWithKeyword() throws Exception { .keyword("keyword").pagination(new Pagination().pageSize(100).rowOffset(0)); List expectedWorkspaces = List.of(generateWorkspace(), generateWorkspace()); - when(workspacePersistence.listWorkspacesByOrganizationId(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), + when(workspacePersistence.listWorkspacesByOrganizationIdPaginated(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), Optional.of("keyword"))) .thenReturn(expectedWorkspaces); WorkspaceReadList result = workspacesHandler.listWorkspacesInOrganization(request); - assertEquals(result.getWorkspaces().size(), 2); + assertEquals(2, result.getWorkspaces().size()); } } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java index c27a48a135a..08acf424019 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java @@ -30,6 +30,7 @@ import io.airbyte.config.ConnectorRegistrySourceDefinition; import io.airbyte.config.ConnectorReleases; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ActorDefinitionVersionResolver; @@ -109,6 +110,7 @@ void testDefaultDefinitionVersionFromCreate() throws IOException { .withDockerRepository(DOCKER_REPOSITORY) .withSpec(new ConnectorSpecification().withProtocolVersion(VALID_PROTOCOL_VERSION)) .withDocumentationUrl(DOCUMENTATION_URL.toString()) + .withSupportLevel(SupportLevel.NONE) .withProtocolVersion(VALID_PROTOCOL_VERSION) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java index aa578485661..6fd46bdfc94 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java @@ -49,6 +49,8 @@ class AutoPropagateSchemaChangeHelperTest { } """; + private static final List SUPPORTED_DESTINATION_SYNC_MODES = List.of( + DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND, DestinationSyncMode.APPEND_DEDUP); private FeatureFlagClient featureFlagClient; @BeforeEach @@ -106,8 +108,9 @@ void applyUpdate() { .transformType(StreamTransform.TransformTypeEnum.UPDATE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(1); Assertions.assertThat(result.getStreams().get(0).getStream().getJsonSchema()).isEqualTo(newSchema); @@ -126,8 +129,9 @@ void applyAddNoFlag() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(2); Assertions.assertThat(result.getStreams().get(0).getStream().getName()).isEqualTo(NAME1); @@ -151,8 +155,9 @@ void applyAdd() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(2); final var stream0 = result.getStreams().get(0); @@ -182,8 +187,9 @@ void applyAddWithSourceDefinedCursor() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(2); final var stream0 = result.getStreams().get(0); @@ -213,8 +219,9 @@ void applyAddWithSourceDefinedCursorNoPrimaryKey() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(2); final var stream1 = result.getStreams().get(1); @@ -237,8 +244,9 @@ void applyAddWithSourceDefinedCursorNoPrimaryKeyNoFullRefresh() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(2); final var stream1 = result.getStreams().get(1); @@ -258,8 +266,9 @@ void applyRemove() { .transformType(StreamTransform.TransformTypeEnum.REMOVE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(0); } @@ -277,8 +286,9 @@ void applyAddNotFully() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(1); Assertions.assertThat(result.getStreams().get(0).getStream().getName()).isEqualTo(NAME1); @@ -297,8 +307,9 @@ void applyRemoveNotFully() { .transformType(StreamTransform.TransformTypeEnum.REMOVE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, featureFlagClient, - UUID.randomUUID()); + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, + UUID.randomUUID()).catalog(); Assertions.assertThat(result.getStreams()).hasSize(1); Assertions.assertThat(result.getStreams().get(0).getStream().getName()).isEqualTo(NAME1); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ConnectorDefinitionSpecificationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ConnectorDefinitionSpecificationHandlerTest.java new file mode 100644 index 00000000000..f269e5a767c --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ConnectorDefinitionSpecificationHandlerTest.java @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers.helpers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; +import io.airbyte.api.model.generated.DestinationIdRequestBody; +import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; +import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; +import io.airbyte.api.model.generated.SourceIdRequestBody; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.lang.Exceptions; +import io.airbyte.commons.server.converters.JobConverter; +import io.airbyte.commons.server.handlers.ConnectorDefinitionSpecificationHandler; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link ConnectorDefinitionSpecificationHandler}. + */ +class ConnectorDefinitionSpecificationHandlerTest { + + private ConfigRepository configRepository; + private ActorDefinitionVersionHelper actorDefinitionVersionHelper; + private JobConverter jobConverter; + private ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler; + + private static final String DESTINATION_DOCKER_TAG = "tag"; + private static final String NAME = "name"; + private static final String SOURCE_DOCKER_REPO = "srcimage"; + private static final String SOURCE_DOCKER_TAG = "tag"; + + private static final ConnectorSpecification CONNECTOR_SPECIFICATION = new ConnectorSpecification() + .withDocumentationUrl(Exceptions.toRuntime(() -> new URI("https://google.com"))) + .withChangelogUrl(Exceptions.toRuntime(() -> new URI("https://google.com"))) + .withConnectionSpecification(Jsons.jsonNode(new HashMap<>())); + + private static final ConnectorSpecification CONNECTOR_SPECIFICATION_WITHOUT_DOCS_URL = new ConnectorSpecification() + .withChangelogUrl(Exceptions.toRuntime(() -> new URI("https://google.com"))) + .withConnectionSpecification(Jsons.jsonNode(new HashMap<>())); + + @BeforeEach + void setup() { + configRepository = mock(ConfigRepository.class); + actorDefinitionVersionHelper = mock(ActorDefinitionVersionHelper.class); + jobConverter = mock(JobConverter.class); + + connectorDefinitionSpecificationHandler = + new ConnectorDefinitionSpecificationHandler(configRepository, actorDefinitionVersionHelper, jobConverter); + } + + @Test + void testGetDestinationSpecForDestinationId() throws JsonValidationException, IOException, ConfigNotFoundException { + final UUID destinationId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + final UUID destinationDefinitionId = UUID.randomUUID(); + final DestinationIdRequestBody destinationIdRequestBody = + new DestinationIdRequestBody() + .destinationId(destinationId); + + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withName(NAME) + .withDestinationDefinitionId(destinationDefinitionId); + when(configRepository.getDestinationConnection(destinationId)).thenReturn( + new DestinationConnection() + .withDestinationId(destinationId) + .withWorkspaceId(workspaceId) + .withDestinationDefinitionId(destinationDefinitionId)); + when(configRepository.getStandardDestinationDefinition(destinationDefinitionId)) + .thenReturn(destinationDefinition); + when(actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, workspaceId, destinationId)) + .thenReturn(new ActorDefinitionVersion() + .withDockerImageTag(DESTINATION_DOCKER_TAG) + .withSpec(CONNECTOR_SPECIFICATION)); + + final DestinationDefinitionSpecificationRead response = + connectorDefinitionSpecificationHandler.getSpecificationForDestinationId(destinationIdRequestBody); + + verify(actorDefinitionVersionHelper).getDestinationVersion(destinationDefinition, workspaceId, destinationId); + assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); + } + + @Test + void testGetSourceSpecWithoutDocs() throws JsonValidationException, IOException, ConfigNotFoundException { + final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId = + new SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); + + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withName(NAME) + .withSourceDefinitionId(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); + when(configRepository.getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId())) + .thenReturn(sourceDefinition); + when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId())) + .thenReturn(new ActorDefinitionVersion() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSpec(CONNECTOR_SPECIFICATION_WITHOUT_DOCS_URL)); + + final SourceDefinitionSpecificationRead response = + connectorDefinitionSpecificationHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId); + + verify(configRepository).getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); + verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); + assertEquals(CONNECTOR_SPECIFICATION_WITHOUT_DOCS_URL.getConnectionSpecification(), response.getConnectionSpecification()); + } + + @Test + void testGetSourceSpecForSourceId() throws JsonValidationException, IOException, ConfigNotFoundException { + final UUID sourceId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + final UUID sourceDefinitionId = UUID.randomUUID(); + + final SourceIdRequestBody sourceIdRequestBody = + new SourceIdRequestBody() + .sourceId(sourceId); + + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withName(NAME) + .withSourceDefinitionId(sourceDefinitionId); + when(configRepository.getSourceConnection(sourceId)).thenReturn( + new SourceConnection() + .withSourceId(sourceId) + .withWorkspaceId(workspaceId) + .withSourceDefinitionId(sourceDefinitionId)); + when(configRepository.getStandardSourceDefinition(sourceDefinitionId)) + .thenReturn(sourceDefinition); + when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, workspaceId, sourceId)) + .thenReturn(new ActorDefinitionVersion() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSpec(CONNECTOR_SPECIFICATION)); + + final SourceDefinitionSpecificationRead response = connectorDefinitionSpecificationHandler.getSpecificationForSourceId(sourceIdRequestBody); + + verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, workspaceId, sourceId); + assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); + } + + @Test + void testGetDestinationSpec() throws JsonValidationException, IOException, ConfigNotFoundException { + final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId = + new DestinationDefinitionIdWithWorkspaceId().destinationDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); + + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withName(NAME) + .withDestinationDefinitionId(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId()); + when(configRepository.getStandardDestinationDefinition(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId())) + .thenReturn(destinationDefinition); + when(actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, + destinationDefinitionIdWithWorkspaceId.getWorkspaceId())) + .thenReturn(new ActorDefinitionVersion() + .withDockerImageTag(DESTINATION_DOCKER_TAG) + .withSpec(CONNECTOR_SPECIFICATION)); + + final DestinationDefinitionSpecificationRead response = + connectorDefinitionSpecificationHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId); + + verify(configRepository).getStandardDestinationDefinition(destinationDefinitionIdWithWorkspaceId.getDestinationDefinitionId()); + verify(actorDefinitionVersionHelper).getDestinationVersion(destinationDefinition, + destinationDefinitionIdWithWorkspaceId.getWorkspaceId()); + assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); + } + + @Test + void testGetSourceSpec() throws JsonValidationException, IOException, ConfigNotFoundException { + final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId = + new SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(UUID.randomUUID()).workspaceId(UUID.randomUUID()); + + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withName(NAME) + .withSourceDefinitionId(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); + when(configRepository.getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId())) + .thenReturn(sourceDefinition); + when(actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId())) + .thenReturn(new ActorDefinitionVersion() + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSpec(CONNECTOR_SPECIFICATION)); + + final SourceDefinitionSpecificationRead response = + connectorDefinitionSpecificationHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId); + + verify(configRepository).getStandardSourceDefinition(sourceDefinitionIdWithWorkspaceId.getSourceDefinitionId()); + verify(actorDefinitionVersionHelper).getSourceVersion(sourceDefinition, sourceDefinitionIdWithWorkspaceId.getWorkspaceId()); + assertEquals(CONNECTOR_SPECIFICATION.getConnectionSpecification(), response.getConnectionSpecification()); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java deleted file mode 100644 index f050bee3ebc..00000000000 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class DefaultInstanceConfigurationHandlerTest { - - private static final String WEBAPP_URL = "http://localhost:8000"; - - private DefaultInstanceConfigurationHandler defaultInstanceConfigurationHandler; - - @BeforeEach - void setup() { - defaultInstanceConfigurationHandler = new DefaultInstanceConfigurationHandler(WEBAPP_URL); - } - - @Test - void testGetInstanceConfiguration() { - final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() - .edition(EditionEnum.COMMUNITY) - .licenseType(null) - .auth(null) - .webappUrl(WEBAPP_URL); - - final InstanceConfigurationResponse actual = defaultInstanceConfigurationHandler.getInstanceConfiguration(); - - assertEquals(expected, actual); - } - -} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java deleted file mode 100644 index acfd9a9cd37..00000000000 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.api.model.generated.AuthConfiguration; -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; -import io.airbyte.commons.license.ActiveAirbyteLicense; -import io.airbyte.commons.license.AirbyteLicense; -import io.airbyte.commons.license.AirbyteLicense.LicenseType; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class ProInstanceConfigurationHandlerTest { - - private static final String WEBAPP_URL = "http://localhost:8000"; - private static final String AIRBYTE_REALM = "airbyte"; - private static final String WEB_CLIENT_ID = "airbyte-webapp"; - - private ProInstanceConfigurationHandler proInstanceConfigurationHandler; - - @BeforeEach - void setup() { - final ActiveAirbyteLicense activeAirbyteLicense = new ActiveAirbyteLicense(); - activeAirbyteLicense.setLicense(new AirbyteLicense(LicenseType.PRO)); - - final AirbyteKeycloakConfiguration keycloakConfiguration = new AirbyteKeycloakConfiguration(); - keycloakConfiguration.setAirbyteRealm(AIRBYTE_REALM); - keycloakConfiguration.setWebClientId(WEB_CLIENT_ID); - - proInstanceConfigurationHandler = new ProInstanceConfigurationHandler(keycloakConfiguration, activeAirbyteLicense, WEBAPP_URL); - } - - @Test - void testGetInstanceConfiguration() { - final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.PRO) - .licenseType(InstanceConfigurationResponse.LicenseTypeEnum.PRO) - .auth(new AuthConfiguration() - .clientId(WEB_CLIENT_ID) - .defaultRealm(AIRBYTE_REALM)) - .webappUrl(WEBAPP_URL); - - final InstanceConfigurationResponse actual = proInstanceConfigurationHandler.getInstanceConfiguration(); - - assertEquals(expected, actual); - } - -} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java new file mode 100644 index 00000000000..3b6d89c8034 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationFields.WORKSPACE_IDS_FIELD_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link AirbyteHttpRequestFieldExtractor} class. + */ +class AirbyteHttpRequestFieldExtractorTest { + + private static final String OTHER_ID = "other_id"; + private static final String SOME_ID = "some_id"; + + private AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor; + + @BeforeEach + void setup() { + airbyteHttpRequestFieldExtractor = new AirbyteHttpRequestFieldExtractor(); + } + + @Test + void testExtractionUUID() { + final UUID match = UUID.randomUUID(); + final UUID other = UUID.randomUUID(); + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isPresent()); + assertEquals(match, UUID.fromString(extractedId.get())); + } + + @Test + void testExtractionNonUUID() { + final Long match = 12345L; + final Long other = Long.MAX_VALUE; + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isPresent()); + assertEquals(match, Long.valueOf(extractedId.get())); + } + + @Test + void testExtractionWithEmptyContent() { + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId("", SOME_ID); + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithMissingField() { + final UUID match = UUID.randomUUID(); + final UUID other = UUID.randomUUID(); + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, "unknownFieldId"); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithNoMatch() { + final String idFieldName = SOME_ID; + final Map content = Map.of(OTHER_ID, UUID.randomUUID().toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithJsonParsingException() { + final String idFieldName = SOME_ID; + final String contentAsString = "{ \"someInvalidJson\" : \" ], \"foo\" }"; + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testWorkspaceIdsExtractionWithNullWorkspaceIds() { + final String idFieldName = WORKSPACE_IDS_FIELD_NAME; + final UUID other = UUID.randomUUID(); + + final Map content = Map.of(OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testWorkspaceIdsExtraction() { + final String idFieldName = WORKSPACE_IDS_FIELD_NAME; + final List valueList = List.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()); + final String valueString = String.format("[\"%s\",\"%s\",\"%s\"]", valueList.get(0), valueList.get(1), valueList.get(2)); + final UUID other = UUID.randomUUID(); + + final Map content = Map.of(WORKSPACE_IDS_FIELD_NAME, valueList, OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertEquals(extractedId, Optional.of(valueString)); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java new file mode 100644 index 00000000000..4cc2c2f5851 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.micronaut.context.event.BeanCreatedEvent; +import io.micronaut.http.server.netty.NettyServerCustomizer; +import io.micronaut.http.server.netty.NettyServerCustomizer.Registry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** + * Test suite for the {@link AuthNettyServerCustomizer} class. + */ +class AuthNettyServerCustomizerTest { + + private static final Integer MAX_CONTENT_LENGTH = 1024; + + private AuthorizationServerHandler authorizationServerHandler; + + private AuthNettyServerCustomizer customizer; + + @BeforeEach + void setup() { + authorizationServerHandler = Mockito.mock(AuthorizationServerHandler.class); + customizer = new AuthNettyServerCustomizer(authorizationServerHandler, MAX_CONTENT_LENGTH); + } + + @Test + void testCustomizerRegisteredOnCreation() { + final BeanCreatedEvent event = mock(BeanCreatedEvent.class); + final Registry registry = mock(Registry.class); + when(event.getBean()).thenReturn(registry); + + final Registry result = customizer.onCreated(event); + assertEquals(registry, result); + verify(registry, times(1)).register(any(NettyServerCustomizer.class)); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java new file mode 100644 index 00000000000..06c1ed7691f --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.persistence.job.WorkspaceHelper; +import io.airbyte.validation.json.JsonValidationException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class AuthenticationHeaderResolverTest { + + @Test + void testResolvingFromWorkspaceId() { + final UUID workspaceId = UUID.randomUUID(); + final Map properties = Map.of(WORKSPACE_ID_HEADER, workspaceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromConnectionId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID connectionId = UUID.randomUUID(); + final Map properties = Map.of(CONNECTION_ID_HEADER, connectionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnectionId(connectionId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceAndDestinationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); + final Map properties = Map.of(DESTINATION_ID_HEADER, destinationId.toString(), SOURCE_ID_HEADER, sourceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnection(sourceId, destinationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromDestinationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); + final Map properties = Map.of(DESTINATION_ID_HEADER, destinationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForDestinationId(destinationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromJobId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final Long jobId = System.currentTimeMillis(); + final Map properties = Map.of(JOB_ID_HEADER, String.valueOf(jobId)); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForJobId(jobId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); + final Map properties = Map.of(SOURCE_ID_HEADER, sourceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForSourceId(sourceId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceDefinitionId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID sourceDefinitionId = UUID.randomUUID(); + final Map properties = Map.of(SOURCE_DEFINITION_ID_HEADER, sourceDefinitionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForSourceId(sourceDefinitionId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromOperationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID operationId = UUID.randomUUID(); + final Map properties = Map.of(OPERATION_ID_HEADER, operationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForOperationId(operationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromNoMatchingProperties() { + final Map properties = Map.of(); + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + final List workspaceId = workspaceResolver.resolveWorkspace(properties); + assertNull(workspaceId); + } + + @Test + void testResolvingWithException() throws JsonValidationException, ConfigNotFoundException { + final UUID connectionId = UUID.randomUUID(); + final Map properties = Map.of(CONNECTION_ID_HEADER, connectionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnectionId(connectionId)).thenThrow(new JsonValidationException("test")); + final List workspaceId = workspaceResolver.resolveWorkspace(properties); + assertNull(workspaceId); + } + + @Test + void testResolvingMultiple() throws JsonValidationException, ConfigNotFoundException { + final List workspaceIds = List.of(UUID.randomUUID(), UUID.randomUUID()); + final Map properties = Map.of(WORKSPACE_IDS_HEADER, Jsons.serialize(workspaceIds)); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List resolvedWorkspaceIds = workspaceResolver.resolveWorkspace(properties); + assertEquals(workspaceIds, resolvedWorkspaceIds); + } + + @Test + void testResolvingOrganizationDirectlyFromHeader() throws ConfigNotFoundException { + final UUID organizationId = UUID.randomUUID(); + final Map properties = Map.of(ORGANIZATION_ID_HEADER, organizationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List result = workspaceResolver.resolveOrganization(properties); + assertEquals(List.of(organizationId), result); + } + + @Test + void testResolvingOrganizationFromWorkspaceHeader() throws ConfigNotFoundException { + final UUID organizationId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + + final Map properties = Map.of(WORKSPACE_ID_HEADER, workspaceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + when(workspaceHelper.getOrganizationForWorkspace(workspaceId)).thenReturn(organizationId); + final List result = workspaceResolver.resolveOrganization(properties); + assertEquals(List.of(organizationId), result); + } + +} diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index bbfbed65fcf..874758fa01c 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -29,6 +29,7 @@ dependencies { compileOnly libs.lombok annotationProcessor libs.lombok implementation libs.bundles.apache + implementation libs.failsafe testImplementation libs.temporal.testing // Needed to be able to mock final class diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java index 25d4c18c285..c79086d7511 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java @@ -4,6 +4,7 @@ package io.airbyte.commons.temporal; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.FAILURE_TYPES_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.TEMPORAL_ACTIVITY_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.TEMPORAL_WORKFLOW_ID_KEY; @@ -67,10 +68,12 @@ public void checkAndHandleCancellation(final Runnable onCancellationCallback) { activityContext.heartbeat(null); } catch (final ActivityCanceledException e) { ApmTraceUtils.addExceptionToTrace(e); + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_TYPES_KEY, e.getClass().getName())); onCancellationCallback.run(); LOGGER.warn("Job was cancelled.", e); } catch (final ActivityCompletionException e) { ApmTraceUtils.addExceptionToTrace(e); + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_TYPES_KEY, e.getClass().getName())); // TODO: This is a hack to avoid having to manually destroy pod, it should be revisited if (!e.getWorkflowId().orElse("").toLowerCase().startsWith("sync")) { LOGGER.warn("The job timeout and was not a sync, we will destroy the pods related to it", e); diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 139b2360dca..9b148bee5c0 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -9,6 +9,10 @@ import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; import io.airbyte.commons.temporal.scheduling.ConnectionUpdaterInput; import io.airbyte.commons.temporal.scheduling.state.WorkflowState; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; @@ -22,26 +26,31 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Function; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; /** * Utility functions for connection manager workflows. */ -@NoArgsConstructor @Singleton @Slf4j public class ConnectionManagerUtils { + private final WorkflowClientWrapped workflowClientWrapped; + private final MetricClient metricClient; + + public ConnectionManagerUtils(final WorkflowClientWrapped workflowClientWrapped, final MetricClient metricClient) { + this.workflowClientWrapped = workflowClientWrapped; + this.metricClient = metricClient; + } + /** * Send a cancellation to the workflow. It will swallow any exception and won't check if the * workflow is already deleted when being cancel. */ - public void deleteWorkflowIfItExist(final WorkflowClient client, - final UUID connectionId) { + public void deleteWorkflowIfItExist(final UUID connectionId) { try { final ConnectionManagerWorkflow connectionManagerWorkflow = - client.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); + workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); connectionManagerWorkflow.deleteConnection(); } catch (final Exception e) { log.warn("The workflow is not reachable when trying to cancel it", e); @@ -56,18 +65,16 @@ public void deleteWorkflowIfItExist(final WorkflowClient client, * batched request. Batching is used to avoid race conditions between starting the workflow and * executing the signal. * - * @param client the WorkflowClient for interacting with temporal * @param connectionId the connection ID to execute this operation for * @param signalMethod a function that takes in a connection manager workflow and executes a signal * method on it, with no arguments * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function signalMethod) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty()); + return signalWorkflowAndRepairIfNecessary(connectionId, signalMethod, Optional.empty()); } /** @@ -77,7 +84,6 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl * batched request. Batching is used to avoid race conditions between starting the workflow and * executing the signal. * - * @param client the WorkflowClient for interacting with temporal * @param connectionId the connection ID to execute this operation for * @param signalMethod a function that takes in a connection manager workflow and executes a signal * method on it, with 1 argument @@ -85,12 +91,11 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function> signalMethod, final T signalArgument) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument)); + return signalWorkflowAndRepairIfNecessary(connectionId, signalMethod, Optional.of(signalArgument)); } // This method unifies the logic of the above two, by using the optional signalArgument parameter to @@ -99,13 +104,12 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Wo // type enforcement for external calls, and means this method can assume consistent type // implementations for both cases. @SuppressWarnings({"MissingJavadocMethod", "LineLength"}) - private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function signalMethod, final Optional signalArgument) throws DeletedWorkflowException { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(connectionId); log.info("Retrieved existing connection manager workflow for connection {}. Executing signal.", connectionId); // retrieve the signal from the lambda final TemporalFunctionalInterfaceMarker signal = signalMethod.apply(connectionManagerWorkflow); @@ -117,6 +121,8 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W } return connectionManagerWorkflow; } catch (final UnreachableWorkflowException e) { + metricClient.count(OssMetricsRegistry.WORFLOW_UNREACHABLE, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); log.error( String.format( "Failed to retrieve ConnectionManagerWorkflow for connection %s. " @@ -126,12 +132,12 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W // in case there is an existing workflow in a bad state, attempt to terminate it first before // starting a new workflow - safeTerminateWorkflow(client, connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); + safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); - final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(connectionId); final ConnectionUpdaterInput startWorkflowInput = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); - final BatchRequest batchRequest = client.newSignalWithStartRequest(); + final BatchRequest batchRequest = workflowClientWrapped.newSignalWithStartRequest(); batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput); // retrieve the signal from the lambda @@ -143,17 +149,17 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W batchRequest.add((Proc) signal); } - client.signalWithStart(batchRequest); + workflowClientWrapped.signalWithStart(batchRequest); log.info("Connection manager workflow for connection {} has been started and signaled.", connectionId); return connectionManagerWorkflow; } } - void safeTerminateWorkflow(final WorkflowClient client, final String workflowId, final String reason) { + void safeTerminateWorkflow(final String workflowId, final String reason) { log.info("Attempting to terminate existing workflow for workflowId {}.", workflowId); try { - client.newUntypedWorkflowStub(workflowId).terminate(reason); + workflowClientWrapped.terminateWorkflow(workflowId, reason); } catch (final Exception e) { log.warn( "Could not terminate temporal workflow due to the following error; " @@ -165,25 +171,23 @@ void safeTerminateWorkflow(final WorkflowClient client, final String workflowId, /** * Terminate a temporal workflow and throw a useful exception. * - * @param client temporal workflow client * @param connectionId connection id * @param reason reason for terminating the workflow */ // todo (cgardens) - what makes this safe - public void safeTerminateWorkflow(final WorkflowClient client, final UUID connectionId, final String reason) { - safeTerminateWorkflow(client, getConnectionManagerName(connectionId), reason); + public void safeTerminateWorkflow(final UUID connectionId, final String reason) { + safeTerminateWorkflow(getConnectionManagerName(connectionId), reason); } /** * Start a connection manager workflow for a connection. * - * @param client temporal workflow client * @param connectionId connection id * @return new connection manager workflow */ // todo (cgardens) - what does no signal mean in this context? - public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { - final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); + public ConnectionManagerWorkflow startConnectionManagerNoSignal(final UUID connectionId) { + final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(connectionId); final ConnectionUpdaterInput input = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); WorkflowClient.start(connectionManagerWorkflow::run, input); @@ -198,16 +202,16 @@ public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowCl * @throws DeletedWorkflowException if the workflow was deleted, according to the workflow state * @throws UnreachableWorkflowException if the workflow is in an unreachable state */ - public ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClient client, final UUID connectionId) + public ConnectionManagerWorkflow getConnectionManagerWorkflow(final UUID connectionId) throws DeletedWorkflowException, UnreachableWorkflowException { final ConnectionManagerWorkflow connectionManagerWorkflow; final WorkflowState workflowState; final WorkflowExecutionStatus workflowExecutionStatus; try { - connectionManagerWorkflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); + connectionManagerWorkflow = workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); workflowState = connectionManagerWorkflow.getState(); - workflowExecutionStatus = getConnectionManagerWorkflowStatus(client, connectionId); + workflowExecutionStatus = getConnectionManagerWorkflowStatus(connectionId); } catch (final Exception e) { throw new UnreachableWorkflowException( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s due to the following error:", connectionId), @@ -229,9 +233,9 @@ public ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClie return connectionManagerWorkflow; } - Optional getWorkflowState(final WorkflowClient client, final UUID connectionId) { + Optional getWorkflowState(final UUID connectionId) { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, + final ConnectionManagerWorkflow connectionManagerWorkflow = workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); return Optional.of(connectionManagerWorkflow.getState()); } catch (final Exception e) { @@ -240,26 +244,25 @@ Optional getWorkflowState(final WorkflowClient client, final UUID } } - boolean isWorkflowStateRunning(final WorkflowClient client, final UUID connectionId) { - return getWorkflowState(client, connectionId).map(WorkflowState::isRunning).orElse(false); + boolean isWorkflowStateRunning(final UUID connectionId) { + return getWorkflowState(connectionId).map(WorkflowState::isRunning).orElse(false); } /** * Get status of a connection manager workflow. * - * @param workflowClient workflow client * @param connectionId connection id * @return workflow execution status */ - public WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final WorkflowClient workflowClient, final UUID connectionId) { + private WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final UUID connectionId) { final DescribeWorkflowExecutionRequest describeWorkflowExecutionRequest = DescribeWorkflowExecutionRequest.newBuilder() .setExecution(WorkflowExecution.newBuilder() .setWorkflowId(getConnectionManagerName(connectionId)) .build()) - .setNamespace(workflowClient.getOptions().getNamespace()).build(); + .setNamespace(workflowClientWrapped.getNamespace()).build(); - final DescribeWorkflowExecutionResponse describeWorkflowExecutionResponse = workflowClient.getWorkflowServiceStubs().blockingStub() - .describeWorkflowExecution(describeWorkflowExecutionRequest); + final DescribeWorkflowExecutionResponse describeWorkflowExecutionResponse = + workflowClientWrapped.blockingDescribeWorkflowExecution(describeWorkflowExecutionRequest); return describeWorkflowExecutionResponse.getWorkflowExecutionInfo().getStatus(); } @@ -267,21 +270,20 @@ public WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final Workflow /** * Get the job id for a connection is a workflow is running for it. Otherwise, throws. * - * @param client temporal workflow client * @param connectionId connection id * @return current job id */ - public long getCurrentJobId(final WorkflowClient client, final UUID connectionId) { + public long getCurrentJobId(final UUID connectionId) { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(connectionId); return connectionManagerWorkflow.getJobInformation().getJobId(); } catch (final Exception e) { return ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; } } - public ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final WorkflowClient client, final UUID connectionId) { - return client.newWorkflowStub(ConnectionManagerWorkflow.class, + private ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final UUID connectionId) { + return workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.CONNECTION_UPDATER, getConnectionManagerName(connectionId))); } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/RetryHelper.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/RetryHelper.java new file mode 100644 index 00000000000..011e52b2518 --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/RetryHelper.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import dev.failsafe.function.CheckedSupplier; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.time.Duration; + +/** + * Helper class to provide consistent retry strategy across the temporal wrapper classes. + */ +public class RetryHelper { + + public static final int DEFAULT_MAX_ATTEMPT = 3; + public static final int DEFAULT_BACKOFF_DELAY_IN_MILLIS = 1000; + public static final int DEFAULT_BACKOFF_MAX_DELAY_IN_MILLIS = 10000; + + private final MetricClient metricClient; + private final int maxAttempt; + private final int backoffDelayInMillis; + private final int backoffMaxDelayInMillis; + + public RetryHelper(final MetricClient metricClient, + final int maxAttempt, + final int backoffDelayInMillis, + final int backoffMaxDelayInMillis) { + this.metricClient = metricClient; + this.maxAttempt = maxAttempt; + this.backoffDelayInMillis = backoffDelayInMillis; + this.backoffMaxDelayInMillis = backoffMaxDelayInMillis; + } + + /** + * Where the magic happens. + *

+ * We should only retry errors that are transient GRPC network errors. + *

+ * We should only retry idempotent calls. The caller should be responsible for retrying creates to + * avoid generating additional noise. + */ + public T withRetries(final CheckedSupplier call, final String name) { + final var retry = RetryPolicy.builder() + .handleIf(this::shouldRetry) + .withMaxAttempts(maxAttempt) + .withBackoff(Duration.ofMillis(backoffDelayInMillis), Duration.ofMillis(backoffMaxDelayInMillis)) + .onRetry((a) -> metricClient.count(OssMetricsRegistry.TEMPORAL_API_TRANSIENT_ERROR_RETRY, 1, + new MetricAttribute(MetricTags.ATTEMPT_NUMBER, String.valueOf(a.getAttemptCount())), + new MetricAttribute(MetricTags.FAILURE_ORIGIN, name), + new MetricAttribute(MetricTags.FAILURE_TYPE, a.getLastException().getClass().getName()))) + .build(); + return Failsafe.with(retry).get(call); + } + + private boolean shouldRetry(final Throwable t) { + // We are retrying Status.UNAVAILABLE because it is often sign of an unexpected connection + // termination. + return t instanceof StatusRuntimeException && Status.UNAVAILABLE.equals(((StatusRuntimeException) t).getStatus()); + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 0f91370625e..b9aa4aaea42 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -22,6 +22,10 @@ import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -31,8 +35,6 @@ import io.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsResponse; import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest; import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse; -import io.temporal.client.WorkflowClient; -import io.temporal.serviceclient.WorkflowServiceStubs; import jakarta.inject.Named; import jakarta.inject.Singleton; import java.io.IOException; @@ -70,27 +72,30 @@ public class TemporalClient { private static final int DELAY_BETWEEN_QUERY_MS = 10; private final Path workspaceRoot; - private final WorkflowClient client; - private final WorkflowServiceStubs service; + private final WorkflowClientWrapped workflowClientWrapped; + private final WorkflowServiceStubsWrapped serviceStubsWrapped; private final StreamResetPersistence streamResetPersistence; private final ConnectionManagerUtils connectionManagerUtils; private final NotificationClient notificationClient; private final StreamResetRecordsHelper streamResetRecordsHelper; + private final MetricClient metricClient; public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, - final WorkflowClient client, - final WorkflowServiceStubs service, + final WorkflowClientWrapped workflowClientWrapped, + final WorkflowServiceStubsWrapped serviceStubsWrapped, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, final NotificationClient notificationClient, - final StreamResetRecordsHelper streamResetRecordsHelper) { + final StreamResetRecordsHelper streamResetRecordsHelper, + final MetricClient metricClient) { this.workspaceRoot = workspaceRoot; - this.client = client; - this.service = service; + this.workflowClientWrapped = workflowClientWrapped; + this.serviceStubsWrapped = serviceStubsWrapped; this.streamResetPersistence = streamResetPersistence; this.connectionManagerUtils = connectionManagerUtils; this.notificationClient = notificationClient; this.streamResetRecordsHelper = streamResetRecordsHelper; + this.metricClient = metricClient; } private final Set workflowNames = new HashSet<>(); @@ -106,9 +111,9 @@ public int restartClosedWorkflowByStatus(final WorkflowExecutionStatus execution final Set nonRunningWorkflow = filterOutRunningWorkspaceId(workflowExecutionInfos); nonRunningWorkflow.forEach(connectionId -> { - connectionManagerUtils.safeTerminateWorkflow(client, connectionId, + connectionManagerUtils.safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(connectionId); }); return nonRunningWorkflow.size(); @@ -118,13 +123,13 @@ Set fetchClosedWorkflowsByStatus(final WorkflowExecutionStatus executionSt ByteString token; ListClosedWorkflowExecutionsRequest workflowExecutionsRequest = ListClosedWorkflowExecutionsRequest.newBuilder() - .setNamespace(client.getOptions().getNamespace()) + .setNamespace(workflowClientWrapped.getNamespace()) .build(); final Set workflowExecutionInfos = new HashSet<>(); do { final ListClosedWorkflowExecutionsResponse listOpenWorkflowExecutionsRequest = - service.blockingStub().listClosedWorkflowExecutions(workflowExecutionsRequest); + serviceStubsWrapped.blockingStubListClosedWorkflowExecutions(workflowExecutionsRequest); final WorkflowType connectionManagerWorkflowType = WorkflowType.newBuilder().setName(ConnectionManagerWorkflow.class.getSimpleName()).build(); workflowExecutionInfos.addAll(listOpenWorkflowExecutionsRequest.getExecutionsList().stream() .filter(workflowExecutionInfo -> workflowExecutionInfo.getType() == connectionManagerWorkflowType @@ -135,7 +140,7 @@ Set fetchClosedWorkflowsByStatus(final WorkflowExecutionStatus executionSt workflowExecutionsRequest = ListClosedWorkflowExecutionsRequest.newBuilder() - .setNamespace(client.getOptions().getNamespace()) + .setNamespace(workflowClientWrapped.getNamespace()) .setNextPageToken(token) .build(); @@ -160,11 +165,11 @@ void refreshRunningWorkflow() { ByteString token; ListOpenWorkflowExecutionsRequest openWorkflowExecutionsRequest = ListOpenWorkflowExecutionsRequest.newBuilder() - .setNamespace(client.getOptions().getNamespace()) + .setNamespace(workflowClientWrapped.getNamespace()) .build(); do { final ListOpenWorkflowExecutionsResponse listOpenWorkflowExecutionsRequest = - service.blockingStub().listOpenWorkflowExecutions(openWorkflowExecutionsRequest); + serviceStubsWrapped.blockingStubListOpenWorkflowExecutions(openWorkflowExecutionsRequest); final Set workflowExecutionInfos = listOpenWorkflowExecutionsRequest.getExecutionsList().stream() .map((workflowExecutionInfo -> workflowExecutionInfo.getExecution().getWorkflowId())) .collect(Collectors.toSet()); @@ -173,7 +178,7 @@ void refreshRunningWorkflow() { openWorkflowExecutionsRequest = ListOpenWorkflowExecutionsRequest.newBuilder() - .setNamespace(client.getOptions().getNamespace()) + .setNamespace(workflowClientWrapped.getNamespace()) .setNextPageToken(token) .build(); @@ -203,7 +208,7 @@ public static class ManualOperationResult { } public Optional getWorkflowState(final UUID connectionId) { - return connectionManagerUtils.getWorkflowState(client, connectionId); + return connectionManagerUtils.getWorkflowState(connectionId); } /** @@ -215,7 +220,7 @@ public Optional getWorkflowState(final UUID connectionId) { public ManualOperationResult startNewManualSync(final UUID connectionId) { log.info("Manual sync request"); - if (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)) { + if (connectionManagerUtils.isWorkflowStateRunning(connectionId)) { // TODO Bmoric: Error is running return new ManualOperationResult( Optional.of("A sync is already running for: " + connectionId), @@ -223,7 +228,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { } try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::submitManualSync); } catch (final DeletedWorkflowException e) { log.error("Can't sync a deleted connection.", e); return new ManualOperationResult( @@ -239,11 +244,11 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { Optional.of("Didn't managed to start a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (!connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (!connectionManagerUtils.isWorkflowStateRunning(connectionId)); log.info("end of manual schedule"); - final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(connectionId); return new ManualOperationResult( Optional.empty(), @@ -259,10 +264,10 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { public ManualOperationResult startNewCancellation(final UUID connectionId) { log.info("Manual cancellation request"); - final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(connectionId); try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::cancelJob); } catch (final DeletedWorkflowException e) { log.error("Can't cancel a deleted workflow", e); return new ManualOperationResult( @@ -278,7 +283,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { Optional.of("Didn't manage to cancel a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (connectionManagerUtils.isWorkflowStateRunning(connectionId)); streamResetRecordsHelper.deleteStreamResetRecordsForJob(jobId, connectionId); @@ -312,13 +317,13 @@ public ManualOperationResult resetConnection(final UUID connectionId, } // get the job ID before the reset, defaulting to NON_RUNNING_JOB_ID if workflow is unreachable - final long oldJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long oldJobId = connectionManagerUtils.getCurrentJobId(connectionId); try { if (syncImmediatelyAfter) { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); } else { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::resetConnection); } } catch (final DeletedWorkflowException e) { log.error("Can't reset a deleted workflow", e); @@ -348,7 +353,7 @@ public ManualOperationResult resetConnection(final UUID connectionId, } private Optional getNewJobId(final UUID connectionId, final long oldJobId) { - final long currentJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long currentJobId = connectionManagerUtils.getCurrentJobId(connectionId); if (currentJobId == NON_RUNNING_JOB_ID || currentJobId == oldJobId) { return Optional.empty(); } else { @@ -499,11 +504,11 @@ TemporalResponse execute(final JobRunConfig jobRunConfig, final Supplier< } private T getWorkflowStub(final Class workflowClass, final TemporalJobType jobType) { - return client.newWorkflowStub(workflowClass, TemporalWorkflowUtils.buildWorkflowOptions(jobType)); + return workflowClientWrapped.newWorkflowStub(workflowClass, TemporalWorkflowUtils.buildWorkflowOptions(jobType)); } private T getWorkflowStubWithTaskQueue(final Class workflowClass, final String taskQueue) { - return client.newWorkflowStub(workflowClass, TemporalWorkflowUtils.buildWorkflowOptionsWithTaskQueue(taskQueue)); + return workflowClientWrapped.newWorkflowStub(workflowClass, TemporalWorkflowUtils.buildWorkflowOptionsWithTaskQueue(taskQueue)); } /** @@ -515,7 +520,7 @@ private T getWorkflowStubWithTaskQueue(final Class workflowClass, final S public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); final ConnectionManagerWorkflow connectionManagerWorkflow = - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(connectionId); try { CompletableFuture.supplyAsync(() -> { try { @@ -542,7 +547,7 @@ public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connect * @param connectionId - connectionId to cancel */ public void forceDeleteWorkflow(final UUID connectionId) { - connectionManagerUtils.deleteWorkflowIfItExist(client, connectionId); + connectionManagerUtils.deleteWorkflowIfItExist(connectionId); } public void sendSchemaChangeNotification(final UUID connectionId, @@ -562,15 +567,17 @@ public void sendSchemaChangeNotification(final UUID connectionId, public void update(final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow; try { - connectionManagerWorkflow = connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerWorkflow = connectionManagerUtils.getConnectionManagerWorkflow(connectionId); } catch (final DeletedWorkflowException e) { log.info("Connection {} is deleted, and therefore cannot be updated.", connectionId); return; } catch (final UnreachableWorkflowException e) { + metricClient.count(OssMetricsRegistry.WORFLOW_UNREACHABLE, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); log.error( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s. Repairing state by creating new workflow.", connectionId), e); - connectionManagerUtils.safeTerminateWorkflow(client, connectionId, + connectionManagerUtils.safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); submitConnectionUpdaterAsync(connectionId); return; @@ -590,7 +597,7 @@ private boolean getConnectorJobSucceeded(final ConnectorJobOutput output) { @VisibleForTesting boolean isWorkflowReachable(final UUID connectionId) { try { - connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerUtils.getConnectionManagerWorkflow(connectionId); return true; } catch (final Exception e) { return false; diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java index 3ea1869cb4b..39f8d1f2e5a 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java @@ -275,24 +275,6 @@ protected NamespaceInfo getNamespaceInfo(final WorkflowServiceStubs temporalServ .getNamespaceInfo(); } - /** - * Run a callable. If while it is running the temporal activity is cancelled, the provided callback - * is triggered. - * - * It manages this by regularly calling back to temporal in order to check whether the activity has - * been cancelled. If it is cancelled it calls the callback. - * - * @param callable callable to run with cancellation - * @param activityContext context used to check whether the activity has been cancelled - * @param type of variable returned by the callable - * @return if the callable succeeds without being cancelled, returns the value returned by the - * callable - */ - public T withBackgroundHeartbeat(final Callable callable, - final Supplier activityContext) { - return withBackgroundHeartbeat(null, callable, activityContext); - } - /** * Run a callable. If while it is running the temporal activity is cancelled, the provided callback * is triggered. @@ -344,7 +326,7 @@ public T withBackgroundHeartbeat(final AtomicReference afterCancel try { // Making sure the heartbeat executor is terminated to avoid heartbeat attempt after we're done with // the activity. - if (scheduledExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + if (scheduledExecutor.awaitTermination(HEARTBEAT_SHUTDOWN_GRACE_PERIOD.toSeconds(), TimeUnit.SECONDS)) { log.info("Temporal heartbeating stopped."); } else { // Heartbeat thread failed to stop, we may leak a thread if this happens. @@ -354,6 +336,8 @@ public T withBackgroundHeartbeat(final AtomicReference afterCancel } catch (InterruptedException e) { // We got interrupted while attempting to shutdown the executor. Not much more we can do. log.info("Interrupted while stopping the Temporal heartbeating, continuing the shutdown."); + // Preserve the interrupt status + Thread.currentThread().interrupt(); } } } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java new file mode 100644 index 00000000000..e42b7f5c680 --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import com.google.common.annotations.VisibleForTesting; +import dev.failsafe.function.CheckedSupplier; +import io.airbyte.metrics.lib.MetricClient; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; +import io.temporal.client.BatchRequest; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; + +/** + * Wrapper around a temporal.client.WorkflowClient. The interface is a subset of the methods that we + * used wrapped with specific handling for transient errors. The goal being to avoid spreading those + * error handling across our codebase. + */ +public class WorkflowClientWrapped { + + private final WorkflowClient temporalWorkflowClient; + private final RetryHelper retryHelper; + + public WorkflowClientWrapped(final WorkflowClient workflowClient, final MetricClient metricClient) { + this(workflowClient, + metricClient, + RetryHelper.DEFAULT_MAX_ATTEMPT, + RetryHelper.DEFAULT_BACKOFF_DELAY_IN_MILLIS, + RetryHelper.DEFAULT_BACKOFF_MAX_DELAY_IN_MILLIS); + } + + @VisibleForTesting + WorkflowClientWrapped(final WorkflowClient workflowClient, + final MetricClient metricClient, + final int maxAttempt, + final int backoffDelayInMillis, + final int backoffMaxDelayInMillis) { + this.temporalWorkflowClient = workflowClient; + this.retryHelper = new RetryHelper(metricClient, maxAttempt, backoffDelayInMillis, backoffMaxDelayInMillis); + } + + /** + * Return the namespace of the temporal client. + */ + public String getNamespace() { + return temporalWorkflowClient.getOptions().getNamespace(); + } + + /** + * Creates workflow client stub for a known execution. Use it to send signals or queries to a + * running workflow. + */ + public T newWorkflowStub(final Class workflowInterface, final String workflowId) { + return withRetries(() -> temporalWorkflowClient.newWorkflowStub(workflowInterface, workflowId), "newWorkflowStub"); + } + + /** + * Creates workflow client stub for a known execution. Use it to send signals or queries to a + * running workflow. + */ + public T newWorkflowStub(final Class workflowInterface, final WorkflowOptions workflowOptions) { + return withRetries(() -> temporalWorkflowClient.newWorkflowStub(workflowInterface, workflowOptions), "newWorkflowStub"); + } + + /** + * Returns information about the specified workflow execution. + */ + public DescribeWorkflowExecutionResponse blockingDescribeWorkflowExecution(final DescribeWorkflowExecutionRequest request) { + return withRetries(() -> temporalWorkflowClient.getWorkflowServiceStubs().blockingStub().describeWorkflowExecution(request), + "describeWorkflowExecution"); + } + + /** + * Terminates a workflow execution. Termination is a hard stop of a workflow execution which doesn't + * give workflow code any chance to perform cleanup. + */ + public void terminateWorkflow(final String workflowId, final String reason) { + withRetries(() -> { + temporalWorkflowClient.newUntypedWorkflowStub(workflowId).terminate(reason); + return null; + }, + "terminate"); + } + + /** + * Creates BatchRequest that can be used to signal an existing workflow or start a new one if not + * running. The batch before invocation must contain exactly two operations. One annotated + * with @WorkflowMethod and another with @SignalMethod. + */ + public BatchRequest newSignalWithStartRequest() { + // NOTE we do not retry here because signals are not idempotent + return temporalWorkflowClient.newSignalWithStartRequest(); + } + + /** + * Invoke SignalWithStart operation. + */ + public WorkflowExecution signalWithStart(final BatchRequest batchRequest) { + // NOTE we do not retry here because signals are not idempotent + return temporalWorkflowClient.signalWithStart(batchRequest); + } + + /** + * Where the magic happens. + *

+ * We should only retry errors that are transient GRPC network errors. + *

+ * We should only retry idempotent calls. The caller should be responsible for retrying creates to + * avoid generating additional noise. + */ + private T withRetries(final CheckedSupplier call, final String name) { + return retryHelper.withRetries(call, name); + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrapped.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrapped.java new file mode 100644 index 00000000000..4879ef8abfd --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrapped.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import dev.failsafe.function.CheckedSupplier; +import io.airbyte.metrics.lib.MetricClient; +import io.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsRequest; +import io.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsResponse; +import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest; +import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse; +import io.temporal.serviceclient.WorkflowServiceStubs; + +/** + * Wrapper around a temporal.client.WorkflowServiceStubs. The interface is a subset of the methods + * that we used wrapped with specific handling for transient errors. The goal being to avoid + * spreading those error handling across our codebase. + */ +public class WorkflowServiceStubsWrapped { + + private final WorkflowServiceStubs workflowServiceStubs; + private final RetryHelper retryHelper; + + public WorkflowServiceStubsWrapped(final WorkflowServiceStubs workflowServiceStubs, final MetricClient metricClient) { + this(workflowServiceStubs, + metricClient, + RetryHelper.DEFAULT_MAX_ATTEMPT, + RetryHelper.DEFAULT_BACKOFF_DELAY_IN_MILLIS, + RetryHelper.DEFAULT_BACKOFF_MAX_DELAY_IN_MILLIS); + } + + public WorkflowServiceStubsWrapped(final WorkflowServiceStubs workflowServiceStubs, + final MetricClient metricClient, + final int maxAttempt, + final int backoffDelayInMillis, + final int backoffMaxDelayInMillis) { + this.workflowServiceStubs = workflowServiceStubs; + this.retryHelper = new RetryHelper(metricClient, maxAttempt, backoffDelayInMillis, backoffMaxDelayInMillis); + } + + /** + * ListClosedWorkflowExecutions is a visibility API to list the closed executions in a specific + * namespace. + */ + public ListClosedWorkflowExecutionsResponse blockingStubListClosedWorkflowExecutions(final ListClosedWorkflowExecutionsRequest request) { + return withRetries(() -> workflowServiceStubs.blockingStub().listClosedWorkflowExecutions(request), "listClosedWorkflowExecutions"); + } + + /** + * ListOpenWorkflowExecutions is a visibility API to list the open executions in a specific + * namespace. + */ + public ListOpenWorkflowExecutionsResponse blockingStubListOpenWorkflowExecutions(final ListOpenWorkflowExecutionsRequest request) { + return withRetries(() -> workflowServiceStubs.blockingStub().listOpenWorkflowExecutions(request), "listOpenWorkflowExecutions"); + } + + /** + * Where the magic happens. + *

+ * We should only retry errors that are transient GRPC network errors. + *

+ * We should only retry idempotent calls. The caller should be responsible for retrying creates to + * avoid generating additional noise. + */ + private T withRetries(final CheckedSupplier call, final String name) { + return retryHelper.withRetries(call, name); + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/config/TemporalBeanFactory.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/config/TemporalBeanFactory.java index eb66a14da9b..1ea02a8232e 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/config/TemporalBeanFactory.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/config/TemporalBeanFactory.java @@ -6,6 +6,9 @@ import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.TemporalWorkflowUtils; +import io.airbyte.commons.temporal.WorkflowClientWrapped; +import io.airbyte.commons.temporal.WorkflowServiceStubsWrapped; +import io.airbyte.metrics.lib.MetricClient; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; import io.temporal.client.WorkflowClient; @@ -20,18 +23,34 @@ @Factory public class TemporalBeanFactory { + /** + * WorkflowServiceStubs shouldn't be used directly, use WorkflowServiceStubsWrapped instead. + */ @Singleton - public WorkflowServiceStubs temporalService(final TemporalUtils temporalUtils, final TemporalSdkTimeouts temporalSdkTimeouts) { + WorkflowServiceStubs temporalService(final TemporalUtils temporalUtils, final TemporalSdkTimeouts temporalSdkTimeouts) { return temporalUtils.createTemporalService(temporalSdkTimeouts); } @Singleton - public WorkflowClient workflowClient( - final TemporalUtils temporalUtils, - final WorkflowServiceStubs temporalService) { + public WorkflowServiceStubsWrapped temporalServiceWrapped(final WorkflowServiceStubs workflowServiceStubs, final MetricClient metricClient) { + return new WorkflowServiceStubsWrapped(workflowServiceStubs, metricClient); + } + + /** + * WorkflowClient shouldn't be used directly, use WorkflowClientWrapped instead. + */ + @Singleton + WorkflowClient workflowClient( + final TemporalUtils temporalUtils, + final WorkflowServiceStubs temporalService) { return TemporalWorkflowUtils.createWorkflowClient(temporalService, temporalUtils.getNamespace()); } + @Singleton + public WorkflowClientWrapped workflowClientWrapped(final WorkflowClient workflowClient, final MetricClient metricClient) { + return new WorkflowClientWrapped(workflowClient, metricClient); + } + @Singleton @Named("workspaceRootTemporal") public Path workspaceRoot(@Value("${airbyte.workspace.root}") final String workspaceRoot) { diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/exception/SizeLimitException.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/exception/SizeLimitException.java new file mode 100644 index 00000000000..a825b27fbc4 --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/exception/SizeLimitException.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal.exception; + +/** + * Exception when an activity fails because the output size exceeds Temporal limits. + */ +public class SizeLimitException extends RuntimeException { + + public SizeLimitException(final String message) { + super(message); + } + +} diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java index e46958c6f56..6d3f8a28257 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java @@ -9,6 +9,7 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; /** * Workflow state change listener for testing. Used to verify the behavior of event signals in @@ -25,7 +26,7 @@ public static void reset() { @Override public Queue events(final UUID testId) { if (!events.containsKey(testId)) { - return new LinkedList<>(); + return new ConcurrentLinkedQueue<>(); } return events.get(testId); diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java new file mode 100644 index 00000000000..6a00ca00f5c --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/utils/PayloadChecker.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal.utils; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.temporal.exception.SizeLimitException; + +/** + * Provide validation to detect temporal failures earlier. + *

+ * For example, when an activity returns a result that exceeds temporal payload limit, we may report + * the activity as a success while it may fail further down in the temporal pipeline. The downside + * is that having this fail in temporal means that we are mistakenly reporting the activity as + * successful. + */ +public class PayloadChecker { + + public static final int MAX_PAYLOAD_SIZE_BYTES = 4 * 1024 * 1024; + + /** + * Validate the payload size fits within temporal message size limits. + * + * @param data to validate + * @param type of data + * @return data if size is valid + * @throws SizeLimitException if payload size exceeds temporal limits. + */ + public static T validatePayloadSize(final T data) { + final String serializedData = Jsons.serialize(data); + if (serializedData.length() > MAX_PAYLOAD_SIZE_BYTES) { + throw new SizeLimitException(String.format("Complete result exceeds size limit (%s of %s)", serializedData.length(), MAX_PAYLOAD_SIZE_BYTES)); + } + return data; + } + +} diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java index 27082940882..89fe7bdeed3 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java @@ -4,7 +4,9 @@ package io.airbyte.commons.temporal; -import io.airbyte.commons.temporal.stubs.HeartbeatWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestActivityImplTest; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestWorkflowImpl; import io.temporal.activity.Activity; import io.temporal.activity.ActivityExecutionContext; import io.temporal.client.WorkflowClient; @@ -23,23 +25,23 @@ void testCancellationHandler() { final Worker worker = testEnv.newWorker("task-queue"); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); new CancellationHandler.TemporalCancellationHandler(context).checkAndHandleCancellation(() -> {}); })); testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final TestWorkflow testWorkflow = client.newWorkflowStub( + TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue("task-queue") .build()); - Assertions.assertDoesNotThrow(heartbeatWorkflow::execute); + Assertions.assertDoesNotThrow(testWorkflow::execute); } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 01425b6401a..97951c769e2 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -39,6 +39,7 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.metrics.lib.MetricClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -122,13 +123,15 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - connectionManagerUtils = spy(new ConnectionManagerUtils()); + final var metricClient = mock(MetricClient.class); + final var workflowClientWrapped = new WorkflowClientWrapped(workflowClient, metricClient); + final var workflowServiceStubsWrapped = new WorkflowServiceStubsWrapped(workflowServiceStubs, metricClient); + connectionManagerUtils = spy(new ConnectionManagerUtils(workflowClientWrapped, metricClient)); notificationClient = spy(new NotificationClient(workflowClient)); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = - spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - notificationClient, - streamResetRecordsHelper)); + spy(new TemporalClient(workspaceRoot, workflowClientWrapped, workflowServiceStubsWrapped, + streamResetPersistence, connectionManagerUtils, notificationClient, streamResetRecordsHelper, mock(MetricClient.class))); } @Nested @@ -142,10 +145,11 @@ void init() { mConnectionManagerUtils = mock(ConnectionManagerUtils.class); mNotificationClient = mock(NotificationClient.class); + final var metricClient = mock(MetricClient.class); temporalClient = spy( - new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - mNotificationClient, - streamResetRecordsHelper)); + new TemporalClient(workspaceRoot, new WorkflowClientWrapped(workflowClient, metricClient), + new WorkflowServiceStubsWrapped(workflowServiceStubs, metricClient), streamResetPersistence, + mConnectionManagerUtils, mNotificationClient, streamResetRecordsHelper, metricClient)); } @Test @@ -162,9 +166,8 @@ void testRestartFailed() { .when(temporalClient).filterOutRunningWorkspaceId(workflowIds); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); temporalClient.restartClosedWorkflowByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); - verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(workflowClient), eq(connectionId), - anyString()); - verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId)); + verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(connectionId), anyString()); + verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(connectionId)); } } @@ -310,7 +313,7 @@ class ForceCancelConnection { void testForceCancelConnection() { temporalClient.forceDeleteWorkflow(CONNECTION_ID); - verify(connectionManagerUtils).deleteWorkflowIfItExist(workflowClient, CONNECTION_ID); + verify(connectionManagerUtils).deleteWorkflowIfItExist(CONNECTION_ID); } } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java index f246d27e756..410a319ae0c 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java @@ -10,10 +10,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.airbyte.commons.concurrency.VoidCallable; -import io.airbyte.commons.temporal.stubs.HeartbeatWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestActivityImplTest; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestWorkflowImpl; import io.temporal.activity.Activity; import io.temporal.activity.ActivityCancellationType; import io.temporal.activity.ActivityExecutionContext; @@ -188,14 +191,16 @@ void testHeartbeatWithContext() throws InterruptedException { final Worker worker = testEnv.newWorker(TASK_QUEUE); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); final CountDownLatch latch = new CountDownLatch(2); + final Runnable cancellationCallback = mock(Runnable.class); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); temporalUtils.withBackgroundHeartbeat( + new AtomicReference<>(cancellationCallback), // TODO (itaseski) figure out how to decrease heartbeat intervals using reflection () -> { latch.await(); @@ -209,16 +214,18 @@ void testHeartbeatWithContext() throws InterruptedException { testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final io.airbyte.commons.temporal.stubs.TestWorkflow testWorkflow = client.newWorkflowStub( + io.airbyte.commons.temporal.stubs.TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue(TASK_QUEUE) .build()); // use async execution to avoid blocking the test thread - WorkflowClient.start(heartbeatWorkflow::execute); + WorkflowClient.start(testWorkflow::execute); assertTrue(latch.await(25, TimeUnit.SECONDS)); + // The activity is expected to succeed, we should never call the cancellation callback. + verify(cancellationCallback, never()).run(); } @@ -229,12 +236,12 @@ void testHeartbeatWithContextAndCallbackRef() throws InterruptedException { final Worker worker = testEnv.newWorker(TASK_QUEUE); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); final CountDownLatch latch = new CountDownLatch(2); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); temporalUtils.withBackgroundHeartbeat( // TODO (itaseski) figure out how to decrease heartbeat intervals using reflection @@ -251,14 +258,14 @@ void testHeartbeatWithContextAndCallbackRef() throws InterruptedException { testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final io.airbyte.commons.temporal.stubs.TestWorkflow testWorkflow = client.newWorkflowStub( + io.airbyte.commons.temporal.stubs.TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue(TASK_QUEUE) .build()); // use async execution to avoid blocking the test thread - WorkflowClient.start(heartbeatWorkflow::execute); + WorkflowClient.start(testWorkflow::execute); assertTrue(latch.await(25, TimeUnit.SECONDS)); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java new file mode 100644 index 00000000000..400c679a1d3 --- /dev/null +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.metrics.lib.MetricClient; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; +import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc.WorkflowServiceBlockingStub; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.serviceclient.WorkflowServiceStubs; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class WorkflowClientWrappedTest { + + static class MyWorkflow {} + + private static final int maxAttempt = 3; + private static final int backoffDelayInMillis = 1; + private static final int backoffMaxDelayInMillis = 10; + private MetricClient metricClient; + private WorkflowServiceStubs temporalWorkflowServiceStubs; + private WorkflowServiceBlockingStub temporalWorkflowServiceBlockingStub; + private WorkflowClient temporalWorkflowClient; + private WorkflowClientWrapped workflowClient; + + @BeforeEach + void beforeEach() { + metricClient = mock(MetricClient.class); + temporalWorkflowServiceBlockingStub = mock(WorkflowServiceBlockingStub.class); + temporalWorkflowServiceStubs = mock(WorkflowServiceStubs.class); + when(temporalWorkflowServiceStubs.blockingStub()).thenReturn(temporalWorkflowServiceBlockingStub); + temporalWorkflowClient = mock(WorkflowClient.class); + when(temporalWorkflowClient.getWorkflowServiceStubs()).thenReturn(temporalWorkflowServiceStubs); + workflowClient = new WorkflowClientWrapped(temporalWorkflowClient, metricClient, maxAttempt, backoffDelayInMillis, backoffMaxDelayInMillis); + } + + @Test + void testRetryLogic() { + when(temporalWorkflowClient.newWorkflowStub(any(), anyString())) + .thenThrow(unavailable()); + + assertThrows(StatusRuntimeException.class, () -> workflowClient.newWorkflowStub(MyWorkflow.class, "fail")); + verify(temporalWorkflowClient, times(3)).newWorkflowStub(any(), anyString()); + verify(metricClient, times(2)).count(any(), anyLong(), any()); + } + + @Test + void testNewWorkflowStub() { + final MyWorkflow expected = new MyWorkflow(); + when(temporalWorkflowClient.newWorkflowStub(any(), anyString())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final MyWorkflow actual = workflowClient.newWorkflowStub(MyWorkflow.class, "woot"); + assertEquals(expected, actual); + } + + @Test + void testNewWorkflowStubWithOptions() { + final MyWorkflow expected = new MyWorkflow(); + when(temporalWorkflowClient.newWorkflowStub(any(), (WorkflowOptions) any())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final MyWorkflow actual = workflowClient.newWorkflowStub(MyWorkflow.class, WorkflowOptions.getDefaultInstance()); + assertEquals(expected, actual); + } + + @Test + void testTerminateWorkflow() { + final var workflowStub = mock(WorkflowStub.class); + when(temporalWorkflowClient.newUntypedWorkflowStub(anyString())) + .thenThrow(unavailable()) + .thenReturn(workflowStub); + + workflowClient.terminateWorkflow("workflow", "test terminate"); + verify(temporalWorkflowClient, times(2)).newUntypedWorkflowStub("workflow"); + verify(workflowStub).terminate("test terminate"); + } + + @Test + void testBlockingDescribeWorkflowExecution() { + final DescribeWorkflowExecutionResponse expected = mock(DescribeWorkflowExecutionResponse.class); + when(temporalWorkflowServiceBlockingStub.describeWorkflowExecution(any())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final DescribeWorkflowExecutionResponse actual = workflowClient.blockingDescribeWorkflowExecution(mock(DescribeWorkflowExecutionRequest.class)); + assertEquals(expected, actual); + } + + @Test + void testSignalsAreNotRetried() { + when(temporalWorkflowClient.signalWithStart(any())).thenThrow(unavailable()); + assertThrows(StatusRuntimeException.class, () -> { + final var request = workflowClient.newSignalWithStartRequest(); + workflowClient.signalWithStart(request); + }); + verify(temporalWorkflowClient, times(1)).signalWithStart(any()); + } + + private static StatusRuntimeException unavailable() { + return new StatusRuntimeException(Status.UNAVAILABLE); + } + +} diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrappedTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrappedTest.java new file mode 100644 index 00000000000..0f88b15d98d --- /dev/null +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowServiceStubsWrappedTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.metrics.lib.MetricClient; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsRequest; +import io.temporal.api.workflowservice.v1.ListClosedWorkflowExecutionsResponse; +import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsRequest; +import io.temporal.api.workflowservice.v1.ListOpenWorkflowExecutionsResponse; +import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc.WorkflowServiceBlockingStub; +import io.temporal.serviceclient.WorkflowServiceStubs; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class WorkflowServiceStubsWrappedTest { + + private static final int maxAttempt = 3; + private static final int backoffDelayInMillis = 1; + private static final int backoffMaxDelayInMillis = 10; + private MetricClient metricClient; + private WorkflowServiceStubs temporalWorkflowServiceStubs; + private WorkflowServiceBlockingStub temporalWorkflowServiceBlockingStub; + private WorkflowServiceStubsWrapped serviceStubsWrapped; + + @BeforeEach + void beforeEach() { + metricClient = mock(MetricClient.class); + temporalWorkflowServiceBlockingStub = mock(WorkflowServiceBlockingStub.class); + temporalWorkflowServiceStubs = mock(WorkflowServiceStubs.class); + when(temporalWorkflowServiceStubs.blockingStub()).thenReturn(temporalWorkflowServiceBlockingStub); + serviceStubsWrapped = new WorkflowServiceStubsWrapped(temporalWorkflowServiceStubs, metricClient, maxAttempt, backoffDelayInMillis, + backoffMaxDelayInMillis); + } + + @Test + void testListClosedWorkflowExecutions() { + final var request = ListClosedWorkflowExecutionsRequest.newBuilder().build(); + final var response = ListClosedWorkflowExecutionsResponse.newBuilder().build(); + when(temporalWorkflowServiceBlockingStub.listClosedWorkflowExecutions(request)) + .thenThrow(unavailable()) + .thenReturn(response); + + final var actual = serviceStubsWrapped.blockingStubListClosedWorkflowExecutions(request); + assertEquals(response, actual); + } + + @Test + void testListOpenWorkflowExecutions() { + final var request = ListOpenWorkflowExecutionsRequest.newBuilder().build(); + final var response = ListOpenWorkflowExecutionsResponse.newBuilder().build(); + when(temporalWorkflowServiceBlockingStub.listOpenWorkflowExecutions(request)) + .thenThrow(unavailable()) + .thenReturn(response); + + final var actual = serviceStubsWrapped.blockingStubListOpenWorkflowExecutions(request); + assertEquals(response, actual); + } + + private static StatusRuntimeException unavailable() { + return new StatusRuntimeException(Status.UNAVAILABLE); + } + +} diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java similarity index 70% rename from airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java rename to airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java index 22dd8f3bb3b..a9a616de438 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java @@ -14,15 +14,17 @@ import io.temporal.workflow.WorkflowMethod; import java.time.Duration; -// todo (cgardens) - don't know what this is meant to do. needs javadocs. +/** + * Workflow used for testing cancellations and heartbeats. + */ @SuppressWarnings("MissingJavadocType") @WorkflowInterface -public interface HeartbeatWorkflow { +public interface TestWorkflow { @WorkflowMethod void execute(); - class HeartbeatWorkflowImpl implements HeartbeatWorkflow { + class TestWorkflowImpl implements TestWorkflow { private final ActivityOptions options = ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofDays(1)) @@ -30,28 +32,28 @@ class HeartbeatWorkflowImpl implements HeartbeatWorkflow { .setRetryOptions(TemporalUtils.NO_RETRY) .build(); - private final HeartbeatActivity heartbeatActivity = Workflow.newActivityStub(HeartbeatActivity.class, options); + private final TestHeartbeatActivity testHeartbeatActivity = Workflow.newActivityStub(TestHeartbeatActivity.class, options); @Override public void execute() { - heartbeatActivity.heartbeat(); + testHeartbeatActivity.heartbeat(); } } @ActivityInterface - interface HeartbeatActivity { + interface TestHeartbeatActivity { @ActivityMethod void heartbeat(); } - class HeartbeatActivityImpl implements HeartbeatActivity { + class TestActivityImplTest implements TestHeartbeatActivity { private final Runnable runnable; - public HeartbeatActivityImpl(final Runnable runnable) { + public TestActivityImplTest(final Runnable runnable) { this.runnable = runnable; } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java new file mode 100644 index 00000000000..f4e2d4e15af --- /dev/null +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/utils/PayloadCheckerTest.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.airbyte.commons.temporal.exception.SizeLimitException; +import org.junit.jupiter.api.Test; + +class PayloadCheckerTest { + + record Payload(String data) {} + + @Test + void testValidPayloadSize() { + final Payload p = new Payload("1".repeat(PayloadChecker.MAX_PAYLOAD_SIZE_BYTES - "{\"data\":\"\"}".length())); + assertEquals(p, PayloadChecker.validatePayloadSize(p)); + } + + @Test + void testInvalidPayloadSize() { + final Payload p = new Payload("1".repeat(PayloadChecker.MAX_PAYLOAD_SIZE_BYTES)); + assertThrows(SizeLimitException.class, () -> PayloadChecker.validatePayloadSize(p)); + } + +} diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index a85a86d98d6..0b044fb71d4 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -74,7 +74,7 @@ dependencies { } test { - maxHeapSize = '2g' + maxHeapSize = '4g' useJUnitPlatform { excludeTags("cloud-storage") diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java index a60019974e6..6687288c8d7 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java @@ -4,6 +4,9 @@ package io.airbyte.workers.general; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.concurrency.BoundedConcurrentLinkedQueue; import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ThreadedTimeTracker; @@ -13,8 +16,10 @@ import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; import io.airbyte.metrics.lib.ApmTraceUtils; +import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; @@ -89,6 +94,7 @@ public class BufferedReplicationWorker implements ReplicationWorker { private static final int sourceMaxBufferSize = 1000; private static final int destinationMaxBufferSize = 1000; private static final int observabilityMetricsPeriodInSeconds = 1; + private static final int executorShutdownGracePeriodInSeconds = 10; public BufferedReplicationWorker(final String jobId, final int attempt, @@ -103,14 +109,13 @@ public BufferedReplicationWorker(final String jobId, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.source = source; this.destination = destination; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning, useNewStateMessageProcessing); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.replicationFeatureFlagReader = replicationFeatureFlagReader; this.recordSchemaValidator = recordSchemaValidator; this.syncPersistence = syncPersistence; @@ -134,6 +139,7 @@ public BufferedReplicationWorker(final String jobId, this.processFromDestStopwatch = new Stopwatch(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException { final Map mdc = MDC.getCopyOfContextMap(); @@ -143,7 +149,7 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo try { final ReplicationContext replicationContext = getReplicationContext(syncInput); final ReplicationFeatureFlags flags = replicationFeatureFlagReader.readReplicationFeatureFlags(syncInput); - replicationWorkerHelper.initialize(replicationContext, flags); + replicationWorkerHelper.initialize(replicationContext, flags, jobRoot); // note: resources are closed in the opposite order in which they are declared. thus source will be // closed first (which is what we want). @@ -174,6 +180,20 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo } finally { executors.shutdownNow(); scheduledExecutors.shutdownNow(); + + try { + // Best effort to mark as complete when the Worker is actually done. + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + scheduledExecutors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + if (!executors.isTerminated() || !scheduledExecutors.isTerminated()) { + final MetricClient metricClient = MetricClientFactory.getMetricClient(); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR, 1, + new MetricAttribute(MetricTags.IMPLEMENTATION, "buffered")); + } + } catch (final InterruptedException e) { + // Preserve the interrupt status + Thread.currentThread().interrupt(); + } } if (!cancelled) { @@ -248,13 +268,19 @@ private ReplicationContext getReplicationContext(final StandardSyncInput syncInp @Override public void cancel() { + boolean wasInterrupted = false; + cancelled = true; replicationWorkerHelper.markCancelled(); LOGGER.info("Cancelling replication worker..."); + executors.shutdownNow(); + scheduledExecutors.shutdownNow(); try { - executors.awaitTermination(10, TimeUnit.SECONDS); + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + scheduledExecutors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); } catch (final InterruptedException e) { + wasInterrupted = true; ApmTraceUtils.addExceptionToTrace(e); LOGGER.error("Unable to cancel due to interruption.", e); } @@ -276,6 +302,11 @@ public void cancel() { } replicationWorkerHelper.endOfReplication(); + + if (wasInterrupted) { + // Preserve the interrupt flag if we were interrupted + Thread.currentThread().interrupt(); + } } /** diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index b01a346d41f..7b47ea85978 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -13,6 +13,11 @@ import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; import io.airbyte.metrics.lib.ApmTraceUtils; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.workers.RecordSchemaValidator; @@ -79,6 +84,8 @@ public class DefaultReplicationWorker implements ReplicationWorker { private final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone; private final ReplicationFeatureFlagReader replicationFeatureFlagReader; + private static final int executorShutdownGracePeriodInSeconds = 10; + public DefaultReplicationWorker(final String jobId, final int attempt, final AirbyteSource source, @@ -92,12 +99,11 @@ public DefaultReplicationWorker(final String jobId, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning, useNewStateMessageProcessing); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.source = source; this.destination = destination; this.syncPersistence = syncPersistence; @@ -140,10 +146,9 @@ public final ReplicationOutput run(final StandardSyncInput syncInput, final Path new ReplicationContext(syncInput.getIsReset(), syncInput.getConnectionId(), syncInput.getSourceId(), syncInput.getDestinationId(), Long.parseLong(jobId), attempt, syncInput.getWorkspaceId()); - ApmTraceUtils.addTagsToTrace(replicationContext.connectionId(), jobId, jobRoot); final ReplicationFeatureFlags flags = replicationFeatureFlagReader.readReplicationFeatureFlags(syncInput); - replicationWorkerHelper.initialize(replicationContext, flags); + replicationWorkerHelper.initialize(replicationContext, flags, jobRoot); replicate(jobRoot, syncInput); @@ -219,6 +224,18 @@ private void replicate(final Path jobRoot, LOGGER.error("Sync worker failed.", e); } finally { executors.shutdownNow(); + + try { + // Best effort to mark as complete when the Worker is actually done. + if (!executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS)) { + final MetricClient metricClient = MetricClientFactory.getMetricClient(); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR, 1, + new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + } + } catch (final InterruptedException e) { + // Preserve the interrupt status + Thread.currentThread().interrupt(); + } } } @@ -344,13 +361,17 @@ private static Runnable readFromSrcAndWriteToDstRunnable(final AirbyteSource sou @Override public void cancel() { + boolean wasInterrupted = false; + // Resources are closed in the opposite order they are declared. LOGGER.info("Cancelling replication worker..."); + executors.shutdownNow(); try { - executors.awaitTermination(10, TimeUnit.SECONDS); + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); } catch (final InterruptedException e) { ApmTraceUtils.addExceptionToTrace(e); LOGGER.error("Unable to cancel due to interruption.", e); + wasInterrupted = true; } cancelled.set(true); replicationWorkerHelper.markCancelled(); @@ -372,6 +393,11 @@ public void cancel() { } replicationWorkerHelper.endOfReplication(); + + if (wasInterrupted) { + // Preserve the interrupt flag if we were interrupted + Thread.currentThread().interrupt(); + } } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index a6df0c58a8f..66c96c33607 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -26,12 +26,9 @@ import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.SourceType; -import io.airbyte.featureflag.UseNewStateMessageProcessing; import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -83,6 +80,7 @@ public class ReplicationWorkerFactory { private final AirbyteMessageDataExtractor airbyteMessageDataExtractor; private final FeatureFlagClient featureFlagClient; private final FeatureFlags featureFlags; + private final MetricClient metricClient; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; public ReplicationWorkerFactory( @@ -94,7 +92,8 @@ public ReplicationWorkerFactory( final SyncPersistenceFactory syncPersistenceFactory, final FeatureFlagClient featureFlagClient, final FeatureFlags featureFlags, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, + final MetricClient metricClient) { this.airbyteIntegrationLauncherFactory = airbyteIntegrationLauncherFactory; this.sourceApi = sourceApi; this.sourceDefinitionApi = sourceDefinitionApi; @@ -105,6 +104,7 @@ public ReplicationWorkerFactory( this.featureFlagClient = featureFlagClient; this.featureFlags = featureFlags; + this.metricClient = metricClient; } /** @@ -122,7 +122,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, "get the source definition for feature flag checks"); final HeartbeatMonitor heartbeatMonitor = createHeartbeatMonitor(sourceDefinitionId, sourceDefinitionApi); final HeartbeatTimeoutChaperone heartbeatTimeoutChaperone = createHeartbeatTimeoutChaperone(heartbeatMonitor, - featureFlagClient, syncInput); + featureFlagClient, syncInput, metricClient); final RecordSchemaValidator recordSchemaValidator = createRecordSchemaValidator(syncInput); // Enable concurrent stream reads for testing purposes @@ -139,9 +139,6 @@ public ReplicationWorker create(final StandardSyncInput syncInput, final var airbyteDestination = airbyteIntegrationLauncherFactory.createAirbyteDestination(destinationLauncherConfig, syncInput.getSyncResourceRequirements(), syncInput.getCatalog()); - // TODO MetricClient should be injectable - MetricClientFactory.initialize(MetricEmittingApps.WORKER); - final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); final FieldSelector fieldSelector = @@ -154,7 +151,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, return createReplicationWorker(airbyteSource, airbyteDestination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, heartbeatTimeoutChaperone, featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning); + onReplicationRunning, metricClient); } /** @@ -213,13 +210,14 @@ private static HeartbeatMonitor createHeartbeatMonitor(final UUID sourceDefiniti */ private static HeartbeatTimeoutChaperone createHeartbeatTimeoutChaperone(final HeartbeatMonitor heartbeatMonitor, final FeatureFlagClient featureFlagClient, - final StandardSyncInput syncInput) { + final StandardSyncInput syncInput, + final MetricClient metricClient) { return new HeartbeatTimeoutChaperone(heartbeatMonitor, HeartbeatTimeoutChaperone.DEFAULT_TIMEOUT_CHECK_DURATION, featureFlagClient, syncInput.getWorkspaceId(), syncInput.getConnectionId(), - MetricClientFactory.getMetricClient()); + metricClient); } /** @@ -264,10 +262,10 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final StandardSyncInput syncInput, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper, - final VoidCallable onReplicationRunning) { + final VoidCallable onReplicationRunning, + final MetricClient metricClient) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); - final boolean useNewStateMessageProcessing = featureFlagClient.boolVariation(UseNewStateMessageProcessing.INSTANCE, flagContext); return buildReplicationWorkerInstance( workerImpl, jobRunConfig.getJobId(), @@ -276,8 +274,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new NamespacingMapper( syncInput.getNamespaceDefinition(), syncInput.getNamespaceFormat(), - syncInput.getPrefix(), - useNewStateMessageProcessing), + syncInput.getPrefix()), destination, messageTracker, syncPersistence, @@ -288,7 +285,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou airbyteMessageDataExtractor, replicationEventPublishingHelper, onReplicationRunning, - useNewStateMessageProcessing); + metricClient); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -328,19 +325,17 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final MetricClient metricClient) { if ("buffered".equals(workerImpl)) { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, onReplicationRunning, useNewStateMessageProcessing); + messageEventPublishingHelper, onReplicationRunning); } else { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, onReplicationRunning, useNewStateMessageProcessing); + messageEventPublishingHelper, onReplicationRunning); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index a91c6bcca78..8bb14e5ef39 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -21,7 +21,12 @@ import io.airbyte.config.SyncStats; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.WorkerSourceConfig; -import io.airbyte.featureflag.UseNewStateMessageProcessing; +import io.airbyte.metrics.lib.ApmTraceUtils; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteTraceMessage; @@ -71,16 +76,17 @@ class ReplicationWorkerHelper { private final FieldSelector fieldSelector; private final AirbyteMapper mapper; private final MessageTracker messageTracker; + private final MetricClient metricClient; private final SyncPersistence syncPersistence; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; private final ThreadedTimeTracker timeTracker; private final VoidCallable onReplicationRunning; - private final boolean useNewStateMessageProcessing; private long recordsRead; private StreamDescriptor currentDestinationStream = null; private ReplicationContext replicationContext = null; private ReplicationFeatureFlags replicationFeatureFlags = null; // NOPMD - keeping this as a placeholder private WorkerDestinationConfig destinationConfig = null; + private MetricAttribute[] metricAttrs = new MetricAttribute[0]; // We expect the number of operations on failures to be low, so synchronizedList should be // performant enough. @@ -96,8 +102,7 @@ public ReplicationWorkerHelper( final SyncPersistence syncPersistence, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, final ThreadedTimeTracker timeTracker, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.airbyteMessageDataExtractor = airbyteMessageDataExtractor; this.fieldSelector = fieldSelector; this.mapper = mapper; @@ -106,8 +111,8 @@ public ReplicationWorkerHelper( this.replicationAirbyteMessageEventPublishingHelper = replicationAirbyteMessageEventPublishingHelper; this.timeTracker = timeTracker; this.onReplicationRunning = onReplicationRunning; - this.useNewStateMessageProcessing = useNewStateMessageProcessing; this.recordsRead = 0L; + this.metricClient = MetricClientFactory.getMetricClient(); } public void markCancelled() { @@ -118,10 +123,13 @@ public void markFailed() { hasFailed.set(true); } - public void initialize(final ReplicationContext replicationContext, final ReplicationFeatureFlags replicationFeatureFlags) { + public void initialize(final ReplicationContext replicationContext, final ReplicationFeatureFlags replicationFeatureFlags, final Path jobRoot) { this.replicationContext = replicationContext; this.replicationFeatureFlags = replicationFeatureFlags; this.timeTracker.trackReplicationStartTime(); + this.metricAttrs = toConnectionAttrs(replicationContext); + ApmTraceUtils.addTagsToTrace(replicationContext.connectionId(), replicationContext.attempt().longValue(), + replicationContext.jobId().toString(), jobRoot); } public void startDestination(final AirbyteDestination destination, final StandardSyncInput syncInput, final Path jobRoot) { @@ -223,14 +231,14 @@ AirbyteMessage internalProcessMessageFromSource(final AirbyteMessage sourceRawMe FileUtils.byteCountToDisplaySize(messageTracker.getSyncStatsTracker().getTotalBytesEmitted())); } + if (sourceRawMessage.getType() == Type.STATE) { + metricClient.count(OssMetricsRegistry.STATE_PROCESSED_FROM_SOURCE, 1, metricAttrs); + } + return sourceRawMessage; } - /** - * If you are making changes in this method please read the documentation in - * {@link #processMessageFromSource} first. - */ - Optional processMessageFromSourceNew(final AirbyteMessage sourceRawMessage) { + public Optional processMessageFromSource(final AirbyteMessage sourceRawMessage) { final AirbyteMessage processedMessage = internalProcessMessageFromSource(sourceRawMessage); // internally we always want to deal with the state message we got from the // source, so we only modify the state message after processing it, right before we send it to the @@ -239,52 +247,9 @@ Optional processMessageFromSourceNew(final AirbyteMessage source } - /** - * If you are making changes in this method please read the documentation in - * {@link #processMessageFromSource} first. - */ - private Optional processMessageFromSourceOld(final AirbyteMessage airbyteMessage) { - fieldSelector.filterSelectedFields(airbyteMessage); - fieldSelector.validateSchema(airbyteMessage); - - final AirbyteMessage message = mapper.mapMessage(airbyteMessage); - - messageTracker.acceptFromSource(message); - - if (shouldPublishMessage(airbyteMessage)) { - replicationAirbyteMessageEventPublishingHelper - .publishStatusEvent(new ReplicationAirbyteMessageEvent(AirbyteMessageOrigin.SOURCE, message, replicationContext)); - } - - recordsRead += 1; - - if (recordsRead % 5000 == 0) { - LOGGER.info("Records read: {} ({})", recordsRead, - FileUtils.byteCountToDisplaySize(messageTracker.getSyncStatsTracker().getTotalBytesEmitted())); - } - - return Optional.of(message); - } - - /** - * The behavior of this method depends on the value of {@link #useNewStateMessageProcessing}, which - * is decided by the {@link UseNewStateMessageProcessing} feature flag. The feature flag is being - * used to test a new version of this method which is meant to fix the issue described here - * https://github.com/airbytehq/airbyte/issues/29478. {@link #processMessageFromSourceNew} is the - * new version of the method that is meant to fix the issue. {@link #processMessageFromSourceOld} is - * the original version of the method where the issue is present. - */ - public Optional processMessageFromSource(final AirbyteMessage airbyteMessage) { - if (useNewStateMessageProcessing) { - return processMessageFromSourceNew(airbyteMessage); - } else { - return processMessageFromSourceOld(airbyteMessage); - } - } - @VisibleForTesting void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMessage) { - LOGGER.info("State in ReplicationWorkerHelper from destination: {}", destinationRawMessage); + LOGGER.debug("State in ReplicationWorkerHelper from destination: {}", destinationRawMessage); final StreamDescriptor previousStream = currentDestinationStream; currentDestinationStream = airbyteMessageDataExtractor.extractStreamDescriptor(destinationRawMessage, previousStream); if (currentDestinationStream != null) { @@ -301,6 +266,8 @@ void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMe messageTracker.acceptFromDestination(destinationRawMessage); if (destinationRawMessage.getType() == Type.STATE) { syncPersistence.persist(replicationContext.connectionId(), destinationRawMessage.getState()); + + metricClient.count(OssMetricsRegistry.STATE_PROCESSED_FROM_DESTINATION, 1, metricAttrs); } if (shouldPublishMessage(destinationRawMessage)) { @@ -310,13 +277,8 @@ void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMe } } - /** - * The value of {@link #useNewStateMessageProcessing} is decided by the - * {@link UseNewStateMessageProcessing} feature flag. The feature flag is being used to test a fix - * the issue described here https://github.com/airbytehq/airbyte/issues/29478. - */ public void processMessageFromDestination(final AirbyteMessage destinationRawMessage) { - final AirbyteMessage message = useNewStateMessageProcessing ? mapper.revertMap(destinationRawMessage) : destinationRawMessage; + final AirbyteMessage message = mapper.revertMap(destinationRawMessage); internalProcessMessageFromDestination(message); } @@ -446,4 +408,23 @@ static FailureReason getFailureReason(final Throwable ex, final long jobId, fina } } + private MetricAttribute[] toConnectionAttrs(final ReplicationContext ctx) { + if (ctx == null) { + return new MetricAttribute[0]; + } + + final var attrs = new ArrayList(); + if (ctx.connectionId() != null) { + attrs.add(new MetricAttribute(MetricTags.CONNECTION_ID, ctx.connectionId().toString())); + } + if (ctx.jobId() != null) { + attrs.add(new MetricAttribute(MetricTags.JOB_ID, ctx.jobId().toString())); + } + if (ctx.attempt() != null) { + attrs.add(new MetricAttribute(MetricTags.ATTEMPT_NUMBER, ctx.attempt().toString())); + } + + return attrs.toArray(new MetricAttribute[0]); + } + } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java index d45a7a2e290..c9495b86e78 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/FailureHelper.java @@ -5,6 +5,8 @@ package io.airbyte.workers.helper; import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.temporal.exception.SizeLimitException; import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; @@ -386,9 +388,13 @@ public static FailureReason failureReasonFromWorkflowAndActivity(final String wo * @return failure reason */ public static FailureReason platformFailure(final Throwable t, final Long jobId, final Integer attemptNumber) { + final String externalMessage = + exceptionChainContains(t, SizeLimitException.class) + ? "Size limit exceeded, please check your configuration, this is often related to a high number of streams." + : "Something went wrong within the airbyte platform"; return genericFailure(t, jobId, attemptNumber) .withFailureOrigin(FailureOrigin.AIRBYTE_PLATFORM) - .withExternalMessage("Something went wrong within the airbyte platform"); + .withExternalMessage(externalMessage); } private static Metadata jobAndAttemptMetadata(final Long jobId, final Integer attemptNumber) { @@ -422,4 +428,16 @@ public static List orderedFailures(final Set failu return failures.stream().sorted(compareByTraceAndTimestamp).toList(); } + @VisibleForTesting + static boolean exceptionChainContains(final Throwable t, Class type) { + Throwable tp = t; + while (tp != null) { + if (type.isInstance(tp)) { + return true; + } + tp = tp.getCause(); + } + return false; + } + } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java index af64df475df..ba00e9d974c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java @@ -7,7 +7,9 @@ import static java.lang.Thread.sleep; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.ShouldFailSyncIfHeartbeatFailure; import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; @@ -15,12 +17,14 @@ import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +115,8 @@ public void runWithHeartbeatThread(final CompletableFuture runnableFuture) LOGGER.info("thread status... heartbeat thread: {} , replication thread: {}", heartbeatFuture.isDone(), runnableFuture.isDone()); if (heartbeatFuture.isDone() && !runnableFuture.isDone()) { - if (featureFlagClient.boolVariation(ShouldFailSyncIfHeartbeatFailure.INSTANCE, new Workspace(workspaceId))) { + if (featureFlagClient.boolVariation(ShouldFailSyncIfHeartbeatFailure.INSTANCE, + new Multi(List.of(new Workspace(workspaceId), new Connection(connectionId))))) { runnableFuture.cancel(true); throw new HeartbeatTimeoutException( String.format("Heartbeat has stopped. Heartbeat freshness threshold: %s secs Actual heartbeat age: %s secs", @@ -154,7 +159,13 @@ void monitor() { @Override public void close() throws Exception { if (lazyExecutorService != null) { - lazyExecutorService.shutdown(); + lazyExecutorService.shutdownNow(); + try { + lazyExecutorService.awaitTermination(10, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + // Propagate the status if we were interrupted + Thread.currentThread().interrupt(); + } } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java index cb4eefcddb7..a6ae8fbe6c1 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java @@ -7,7 +7,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.json.Jsons; import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; -import io.airbyte.featureflag.UseNewStateMessageProcessing; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -36,7 +35,6 @@ public class NamespacingMapper implements AirbyteMapper { private final String namespaceFormat; private final String streamPrefix; private final Map destinationToSourceNamespaceAndStreamName; - private final boolean useNewNamespaceMapping; @VisibleForTesting record NamespaceAndStreamName(String namespace, String streamName) {} @@ -44,9 +42,8 @@ record NamespaceAndStreamName(String namespace, String streamName) {} public NamespacingMapper( final NamespaceDefinitionType namespaceDefinition, final String namespaceFormat, - final String streamPrefix, - final boolean useNewNamespaceMapping) { - this(namespaceDefinition, namespaceFormat, streamPrefix, new HashMap<>(), useNewNamespaceMapping); + final String streamPrefix) { + this(namespaceDefinition, namespaceFormat, streamPrefix, new HashMap<>()); } @VisibleForTesting @@ -54,13 +51,11 @@ public NamespacingMapper( final NamespaceDefinitionType namespaceDefinition, final String namespaceFormat, final String streamPrefix, - final Map destinationToSourceNamespaceAndStreamName, - final boolean useNewNamespaceMapping) { + final Map destinationToSourceNamespaceAndStreamName) { this.namespaceDefinition = namespaceDefinition; this.namespaceFormat = namespaceFormat; this.streamPrefix = streamPrefix; this.destinationToSourceNamespaceAndStreamName = destinationToSourceNamespaceAndStreamName; - this.useNewNamespaceMapping = useNewNamespaceMapping; } @Override @@ -85,48 +80,8 @@ public ConfiguredAirbyteCatalog mapCatalog(final ConfiguredAirbyteCatalog inputC return catalog; } - /** - * The behavior of this method depends on the value of {@link #useNewNamespaceMapping}, which is - * decided by the {@link UseNewStateMessageProcessing} feature flag. The feature flag is being used - * to test a new version of this method which is meant to fix the issue described here - * https://github.com/airbytehq/airbyte/issues/29478. {@link #mapMessageNew} is the new version of - * the method that is meant to fix the issue. {@link #mapMessageOld} is the original version of the - * method where the issue is present. - */ @Override public AirbyteMessage mapMessage(final AirbyteMessage message) { - if (useNewNamespaceMapping) { - return mapMessageNew(message); - } else { - return mapMessageOld(message); - } - } - - /** - * If you are making changes in this method please read the documentation in {@link #mapMessage} - * first. - */ - AirbyteMessage mapMessageOld(final AirbyteMessage message) { - if (message.getType() == Type.RECORD) { - // Default behavior if namespaceDefinition is not set is to follow SOURCE - if (namespaceDefinition != null) { - if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) { - message.getRecord().withNamespace(null); - } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) { - message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat)); - } - } - message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix)); - return message; - } - return message; - } - - /** - * If you are making changes in this method please read the documentation in {@link #mapMessage} - * first. - */ - AirbyteMessage mapMessageNew(final AirbyteMessage message) { if (message.getType() == Type.RECORD) { final AirbyteRecordMessage recordMessage = message.getRecord(); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java index 84af91329f1..3f5f485c0ea 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java @@ -6,7 +6,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.functional.CheckedSupplier; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.commons.workers.config.WorkerConfigsProvider; import io.airbyte.commons.workers.config.WorkerConfigsProvider.ResourceType; @@ -45,18 +44,15 @@ public class KubeOrchestratorHandleFactory implements OrchestratorHandleFactory private final ContainerOrchestratorConfig containerOrchestratorConfig; private final WorkerConfigsProvider workerConfigsProvider; private final FeatureFlagClient featureFlagClient; - private final TemporalUtils temporalUtils; private final Integer serverPort; public KubeOrchestratorHandleFactory(@Named("containerOrchestratorConfig") final ContainerOrchestratorConfig containerOrchestratorConfig, final WorkerConfigsProvider workerConfigsProvider, final FeatureFlagClient featureFlagClient, - final TemporalUtils temporalUtils, @Value("${micronaut.server.port}") final Integer serverPort) { this.containerOrchestratorConfig = containerOrchestratorConfig; this.workerConfigsProvider = workerConfigsProvider; this.featureFlagClient = featureFlagClient; - this.temporalUtils = temporalUtils; this.serverPort = serverPort; } @@ -78,9 +74,7 @@ public CheckedSupplier, Exception> destinationLauncherConfig, jobRunConfig, syncInput.getSyncResourceRequirements() != null ? syncInput.getSyncResourceRequirements().getOrchestrator() : null, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient); } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java index 369687bbd51..d09af1922ee 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java @@ -60,6 +60,7 @@ public class AsyncOrchestratorPodProcess implements KubePod { public static final String NO_OP = "NO_OP"; // TODO Ths frequency should be configured and injected rather hard coded here. public static final long JOB_STATUS_POLLING_FREQUENCY_IN_MILLIS = 5000; + private static final String JAVA_OOM_EXCEPTION_STRING = "java.lang.OutOfMemoryError"; private final KubePodInfo kubePodInfo; private final DocumentStoreClient documentStoreClient; @@ -177,6 +178,12 @@ private int computeExitValue() { } else { // otherwise, the actual pod is terminal when the doc store says it shouldn't be. log.info("The current non terminal state is {}", secondDocStoreStatus); + + if (isOOM(pod)) { + log.warn("Terminating due to OutOfMemoryError"); + return 137; + } + log.warn("State Store missing status, however orchestrator pod {} in terminal. Assume failure.", getInfo().name()); return 3; } @@ -190,6 +197,25 @@ private int computeExitValue() { } } + private boolean isOOM(final Pod pod) { + if (pod == null) { + return false; + } + try { + return kubernetesClient.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .withName(pod.getFullResourceName()) + .tailingLines(5) + .getLog() + .contains(JAVA_OOM_EXCEPTION_STRING); + } catch (final Exception e) { + // We are trying to detect if the pod OOMed, if we fail to check the logs, we don't want to add more + // exception noise since this is an extra inspection attempt for more precise error logging. + log.info("Failed to retrieve pod logs for additional debugging information.", e); + } + return false; + } + @Override public int exitValue() { final var optionalCached = cachedExitValue.get(); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index c4761917a58..4eba409a968 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -6,6 +6,7 @@ import autovalue.shaded.org.jetbrains.annotations.NotNull; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.workers.config.WorkerConfigs; @@ -126,7 +127,9 @@ public Process create( final WorkerConfigs workerConfigs = workerConfigsProvider.getConfig(resourceType); - final var allLabels = getLabels(jobId, attempt, connectionId, workspaceId, customLabels, workerConfigs.getWorkerKubeLabels()); + final String shortImageName = imageName != null ? DockerImageNameHelper.extractShortImageName(imageName) : null; + + final var allLabels = getLabels(jobId, attempt, connectionId, workspaceId, shortImageName, customLabels, workerConfigs.getWorkerKubeLabels()); // If using isolated pool, check workerConfigs has isolated pool set. If not set, fall back to use // regular node pool. @@ -168,7 +171,7 @@ public Process create( internalToExternalPorts, args).toProcess(); } catch (final Exception e) { - throw new WorkerException(e.getMessage(), e); + throw new WorkerException("Failed to create pod for " + jobType + " step", e); } } @@ -180,6 +183,7 @@ public static Map getLabels(final String jobId, final int attemptId, final UUID connectionId, final UUID workspaceId, + final String imageName, final Map customLabels, final Map envLabels) { final Map allLabels = new HashMap<>(); @@ -193,7 +197,8 @@ public static Map getLabels(final String jobId, Metadata.ATTEMPT_LABEL_KEY, String.valueOf(attemptId), Metadata.CONNECTION_ID_LABEL_KEY, String.valueOf(connectionId), Metadata.WORKSPACE_LABEL_KEY, String.valueOf(workspaceId), - Metadata.WORKER_POD_LABEL_KEY, Metadata.WORKER_POD_LABEL_VALUE); + Metadata.WORKER_POD_LABEL_KEY, Metadata.WORKER_POD_LABEL_VALUE, + Metadata.IMAGE_NAME, imageName); allLabels.putAll(generalKubeLabels); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java index 7d42a39b2a1..19f0ddf41a5 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java @@ -19,6 +19,7 @@ public final class Metadata { static final String WORKER_POD_LABEL_KEY = "airbyte"; static final String WORKER_POD_LABEL_VALUE = "job-pod"; public static final String CONNECTION_ID_LABEL_KEY = "connection_id"; + static final String IMAGE_NAME = "image_name"; /** * These are more readable forms of {@link io.airbyte.config.JobTypeResourceLimit.JobType}. diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java index 1b70c483805..890b11d0a6c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java @@ -8,17 +8,14 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.OperatorDbtInput; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Dbt Launcher Worker. @@ -35,9 +32,7 @@ public DbtLauncherWorker(final UUID connectionId, final JobRunConfig jobRunConfig, final WorkerConfigs workerConfigs, final ContainerOrchestratorConfig containerOrchestratorConfig, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final FeatureFlagClient featureFlagClient) { super( connectionId, @@ -50,9 +45,7 @@ public DbtLauncherWorker(final UUID connectionId, containerOrchestratorConfig, workerConfigs.getResourceRequirements(), Void.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, // Custom connector does not use Dbt at this moment, thus this flag for runnning job under diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java index e8fd9598322..ee7131c025f 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java @@ -13,9 +13,9 @@ import com.google.common.base.Stopwatch; import datadog.trace.api.Trace; import io.airbyte.commons.constants.WorkerConstants; +import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.sync.OrchestratorConstants; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.ResourceRequirements; @@ -38,7 +38,6 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.KubernetesClientException; import io.micronaut.core.util.StringUtils; -import io.temporal.activity.ActivityExecutionContext; import java.nio.file.Path; import java.time.Duration; import java.util.Collections; @@ -48,8 +47,6 @@ import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; @@ -84,9 +81,7 @@ public abstract class LauncherWorker implements Worker outputClass; - private final Supplier activityContext; private final Integer serverPort; - private final TemporalUtils temporalUtils; private final WorkerConfigs workerConfigs; private final boolean isCustomConnector; @@ -103,9 +98,7 @@ public LauncherWorker(final UUID connectionId, final ContainerOrchestratorConfig containerOrchestratorConfig, final ResourceRequirements resourceRequirements, final Class outputClass, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final WorkerConfigs workerConfigs, final FeatureFlagClient featureFlagClient, final boolean isCustomConnector) { @@ -119,9 +112,7 @@ public LauncherWorker(final UUID connectionId, this.containerOrchestratorConfig = containerOrchestratorConfig; this.resourceRequirements = resourceRequirements; this.outputClass = outputClass; - this.activityContext = activityContext; this.serverPort = serverPort; - this.temporalUtils = temporalUtils; this.workerConfigs = workerConfigs; this.featureFlagClient = featureFlagClient; this.isCustomConnector = isCustomConnector; @@ -133,151 +124,140 @@ public LauncherWorker(final UUID connectionId, @Trace(operationName = WORKER_OPERATION_NAME) @Override public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException { - final AtomicBoolean isCanceled = new AtomicBoolean(false); - final AtomicReference cancellationCallback = new AtomicReference<>(null); - return temporalUtils.withBackgroundHeartbeat(cancellationCallback, () -> { - try { - // Assemble configuration. - final Map envMap = System.getenv().entrySet().stream() - .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - // Manually add the worker environment to the env var map - envMap.put(WorkerConstants.WORKER_ENVIRONMENT, containerOrchestratorConfig.workerEnvironment().name()); - - // Merge in the env from the ContainerOrchestratorConfig - containerOrchestratorConfig.environmentVariables().entrySet().stream().forEach(e -> envMap.putIfAbsent(e.getKey(), e.getValue())); - - // Allow for the override of the socat pod CPU resources as part of the concurrent source read - // experimentation - final String socatResources = featureFlagClient.stringVariation(ConcurrentSocatResources.INSTANCE, new Connection(connectionId)); - if (StringUtils.isNotEmpty(socatResources)) { - LOGGER.info("Overriding Socat CPU limit and request to {}.", socatResources); - envMap.put(SOCAT_KUBE_CPU_LIMIT, socatResources); - envMap.put(SOCAT_KUBE_CPU_REQUEST, socatResources); - } - - final Map fileMap = new HashMap<>(additionalFileMap); - fileMap.putAll(Map.of( - OrchestratorConstants.INIT_FILE_APPLICATION, application, - OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), - OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), - // OrchestratorConstants.INIT_FILE_ENV_MAP might be duplicated since the pod env contains everything - OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); - - final Map portMap = Map.of( - serverPort, serverPort, - OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, - OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, - OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, - OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); - - final var allLabels = KubeProcessFactory.getLabels( - jobRunConfig.getJobId(), - Math.toIntExact(jobRunConfig.getAttemptId()), - connectionId, - workspaceId, - generateMetadataLabels(), - Collections.emptyMap()); - - final var podNameAndJobPrefix = podNamePrefix + "-job-" + jobRunConfig.getJobId() + "-attempt-"; - final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); - final var mainContainerInfo = new KubeContainerInfo(containerOrchestratorConfig.containerOrchestratorImage(), - containerOrchestratorConfig.containerOrchestratorImagePullPolicy()); - final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), - podName, - mainContainerInfo); - - ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getJobId(), jobRoot); - - final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); - - // Use the configuration to create the process. - process = new AsyncOrchestratorPodProcess( - kubePodInfo, - containerOrchestratorConfig.documentStoreClient(), - containerOrchestratorConfig.kubernetesClient(), - containerOrchestratorConfig.secretName(), - containerOrchestratorConfig.secretMountPath(), - containerOrchestratorConfig.dataPlaneCredsSecretName(), - containerOrchestratorConfig.dataPlaneCredsSecretMountPath(), - containerOrchestratorConfig.googleApplicationCredentials(), - envMap, - workerConfigs.getWorkerKubeAnnotations(), - serverPort, - containerOrchestratorConfig.serviceAccount(), - schedulerName.isBlank() ? null : schedulerName); - - // Define what to do on cancellation. - cancellationCallback.set(() -> { - // When cancelled, try to set to true. - // Only proceed if value was previously false, so we only have one cancellation going. at a time - if (!isCanceled.getAndSet(true)) { - log.info("Trying to cancel async pod process."); - process.destroy(); - } - }); - - // only kill running pods and create process if it is not already running. - if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { - log.info("Creating " + podName + " for attempt number: " + jobRunConfig.getAttemptId()); - killRunningPodsForConnection(); - - // custom connectors run in an isolated node pool from airbyte-supported connectors - // to reduce the blast radius of any problems with custom connector code. - final var nodeSelectors = - isCustomConnector ? workerConfigs.getWorkerIsolatedKubeNodeSelectors().orElse(workerConfigs.getworkerKubeNodeSelectors()) - : workerConfigs.getworkerKubeNodeSelectors(); - - try { - process.create( - allLabels, - resourceRequirements, - fileMap, - portMap, - nodeSelectors); - } catch (final KubernetesClientException e) { - ApmTraceUtils.addExceptionToTrace(e); - throw new WorkerException( - "Failed to create pod " + podName + ", pre-existing pod exists which didn't advance out of the NOT_STARTED state.", e); - } - } - - // this waitFor can resume if the activity is re-run - process.waitFor(); + try { + // Assemble configuration. + final Map envMap = System.getenv().entrySet().stream() + .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Manually add the worker environment to the env var map + envMap.put(WorkerConstants.WORKER_ENVIRONMENT, containerOrchestratorConfig.workerEnvironment().name()); + + // Merge in the env from the ContainerOrchestratorConfig + containerOrchestratorConfig.environmentVariables().entrySet().stream().forEach(e -> envMap.putIfAbsent(e.getKey(), e.getValue())); + + // Allow for the override of the socat pod CPU resources as part of the concurrent source read + // experimentation + final String socatResources = featureFlagClient.stringVariation(ConcurrentSocatResources.INSTANCE, new Connection(connectionId)); + if (StringUtils.isNotEmpty(socatResources)) { + LOGGER.info("Overriding Socat CPU limit and request to {}.", socatResources); + envMap.put(SOCAT_KUBE_CPU_LIMIT, socatResources); + envMap.put(SOCAT_KUBE_CPU_REQUEST, socatResources); + } - if (cancelled.get()) { - final CancellationException e = new CancellationException(); + final Map fileMap = new HashMap<>(additionalFileMap); + fileMap.putAll(Map.of( + OrchestratorConstants.INIT_FILE_APPLICATION, application, + OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), + OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), + // OrchestratorConstants.INIT_FILE_ENV_MAP might be duplicated since the pod env contains everything + OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); + + final Map portMap = Map.of( + serverPort, serverPort, + OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, + OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, + OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, + OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); + + final var podNameAndJobPrefix = podNamePrefix + "-job-" + jobRunConfig.getJobId() + "-attempt-"; + final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); + final var mainContainerInfo = new KubeContainerInfo(containerOrchestratorConfig.containerOrchestratorImage(), + containerOrchestratorConfig.containerOrchestratorImagePullPolicy()); + final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), + podName, + mainContainerInfo); + + ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getAttemptId(), jobRunConfig.getJobId(), jobRoot); + + final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); + + final String shortImageName = + mainContainerInfo.image() != null ? DockerImageNameHelper.extractShortImageName(mainContainerInfo.image()) : null; + final var allLabels = KubeProcessFactory.getLabels( + jobRunConfig.getJobId(), + Math.toIntExact(jobRunConfig.getAttemptId()), + connectionId, + workspaceId, + shortImageName, + generateMetadataLabels(), + Collections.emptyMap()); + + // Use the configuration to create the process. + process = new AsyncOrchestratorPodProcess( + kubePodInfo, + containerOrchestratorConfig.documentStoreClient(), + containerOrchestratorConfig.kubernetesClient(), + containerOrchestratorConfig.secretName(), + containerOrchestratorConfig.secretMountPath(), + containerOrchestratorConfig.dataPlaneCredsSecretName(), + containerOrchestratorConfig.dataPlaneCredsSecretMountPath(), + containerOrchestratorConfig.googleApplicationCredentials(), + envMap, + workerConfigs.getWorkerKubeAnnotations(), + serverPort, + containerOrchestratorConfig.serviceAccount(), + schedulerName.isBlank() ? null : schedulerName); + + // only kill running pods and create process if it is not already running. + if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { + log.info("Creating " + podName + " for attempt number: " + jobRunConfig.getAttemptId()); + killRunningPodsForConnection(); + + // custom connectors run in an isolated node pool from airbyte-supported connectors + // to reduce the blast radius of any problems with custom connector code. + final var nodeSelectors = + isCustomConnector ? workerConfigs.getWorkerIsolatedKubeNodeSelectors().orElse(workerConfigs.getworkerKubeNodeSelectors()) + : workerConfigs.getworkerKubeNodeSelectors(); + + try { + process.create( + allLabels, + resourceRequirements, + fileMap, + portMap, + nodeSelectors); + } catch (final KubernetesClientException e) { ApmTraceUtils.addExceptionToTrace(e); - throw e; + throw new WorkerException( + "Failed to create pod " + podName + ", pre-existing pod exists which didn't advance out of the NOT_STARTED state.", e); } + } - final int asyncProcessExitValue = process.exitValue(); - if (asyncProcessExitValue != 0) { - final WorkerException e = new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); - ApmTraceUtils.addTagsToTrace(Map.of(PROCESS_EXIT_VALUE_KEY, asyncProcessExitValue)); - ApmTraceUtils.addExceptionToTrace(e); - throw e; - } + // this waitFor can resume if the activity is re-run + process.waitFor(); - final var output = process.getOutput(); + if (cancelled.get()) { + final CancellationException e = new CancellationException(); + ApmTraceUtils.addExceptionToTrace(e); + throw e; + } - return output.map(s -> Jsons.deserialize(s, outputClass)).orElse(null); - } catch (final Exception e) { + final int asyncProcessExitValue = process.exitValue(); + if (asyncProcessExitValue != 0) { + final WorkerException e = new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); + ApmTraceUtils.addTagsToTrace(Map.of(PROCESS_EXIT_VALUE_KEY, asyncProcessExitValue)); ApmTraceUtils.addExceptionToTrace(e); - if (cancelled.get()) { - try { - log.info("Destroying process due to cancellation."); - process.destroy(); - } catch (final Exception e2) { - log.error("Failed to destroy process on cancellation.", e2); - } - throw new WorkerException("Launcher " + application + " was cancelled.", e); - } else { - throw new WorkerException("Running the launcher " + application + " failed", e); + throw e; + } + + final var output = process.getOutput(); + + return output.map(s -> Jsons.deserialize(s, outputClass)).orElse(null); + } catch (final Exception e) { + ApmTraceUtils.addExceptionToTrace(e); + if (cancelled.get()) { + try { + log.info("Destroying process due to cancellation."); + process.destroy(); + } catch (final Exception e2) { + log.error("Failed to destroy process on cancellation.", e2); } + throw new WorkerException("Launcher " + application + " was cancelled.", e); + } else { + throw new WorkerException("Running the launcher " + application + " failed", e); } - }, activityContext); + } } private Map generateMetadataLabels() { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java index aa4f8a48a5c..d6eb84cce8e 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java @@ -8,7 +8,6 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; @@ -16,10 +15,8 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Normalization Launcher Worker. @@ -36,9 +33,7 @@ public NormalizationLauncherWorker(final UUID connectionId, final JobRunConfig jobRunConfig, final WorkerConfigs workerConfigs, final ContainerOrchestratorConfig containerOrchestratorConfig, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final FeatureFlagClient featureFlagClient) { super( connectionId, @@ -51,9 +46,7 @@ public NormalizationLauncherWorker(final UUID connectionId, containerOrchestratorConfig, workerConfigs.getResourceRequirements(), NormalizationSummary.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, // Normalization process will happen only on a fixed set of connectors, diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java index 2992443ebab..bf35d9ab236 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java @@ -8,7 +8,6 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.ResourceRequirements; @@ -17,10 +16,8 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Launches a container-orchestrator container/pod to manage the message passing for the replication @@ -41,9 +38,7 @@ public ReplicationLauncherWorker(final UUID connectionId, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, final ResourceRequirements resourceRequirements, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final WorkerConfigs workerConfigs, final FeatureFlagClient featureFlagClient) { super( @@ -58,9 +53,7 @@ public ReplicationLauncherWorker(final UUID connectionId, containerOrchestratorConfig, resourceRequirements, ReplicationOutput.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, sourceLauncherConfig.getIsCustomConnector() || destinationLauncherConfig.getIsCustomConnector()); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java index c2b89c7f9c1..6275401f663 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java @@ -37,8 +37,7 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning, - false); + onReplicationRunning); } // BufferedReplicationWorkerTests. diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index a075f282f9b..3423b90f2e4 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -31,8 +31,7 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning, - false); + onReplicationRunning); } // DefaultReplicationWorkerTests. diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java index 168da046cb5..f47ce1a8a0b 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java @@ -34,8 +34,7 @@ void setUp() { null, null, null, - null, - true)); + null)); } @Test @@ -46,7 +45,7 @@ void testMessageIsMappedAfterProcessing() { doReturn(sourceRawMessage).when(replicationWorkerHelper).internalProcessMessageFromSource(sourceRawMessage); when(mapper.mapMessage(sourceRawMessage)).thenReturn(mappedSourceMessage); - final Optional processedMessageFromSource = replicationWorkerHelper.processMessageFromSourceNew(sourceRawMessage); + final Optional processedMessageFromSource = replicationWorkerHelper.processMessageFromSource(sourceRawMessage); assertEquals(Optional.of(mappedSourceMessage), processedMessageFromSource); } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java index 94e18f571c2..663a78ad252 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java @@ -208,6 +208,8 @@ void setup() throws Exception { when(mapper.mapMessage(RECORD_MESSAGE2)).thenReturn(RECORD_MESSAGE2); when(mapper.mapMessage(RECORD_MESSAGE3)).thenReturn(RECORD_MESSAGE3); when(mapper.mapMessage(CONFIG_MESSAGE)).thenReturn(CONFIG_MESSAGE); + when(mapper.revertMap(STATE_MESSAGE)).thenReturn(STATE_MESSAGE); + when(mapper.revertMap(CONFIG_MESSAGE)).thenReturn(CONFIG_MESSAGE); when(heartbeatMonitor.isBeating()).thenReturn(Optional.of(true)); } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java index 921cfa46854..a4b96f1db53 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java @@ -4,7 +4,6 @@ package io.airbyte.workers.general.performance; -import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.general.BufferedReplicationWorker; import io.airbyte.workers.general.ReplicationFeatureFlagReader; @@ -38,11 +37,10 @@ public ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient) { + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, () -> {}, false); + messageEventPublishingHelper, () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java index be9e9b52398..4ba578546b5 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java @@ -4,7 +4,6 @@ package io.airbyte.workers.general.performance; -import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.general.DefaultReplicationWorker; import io.airbyte.workers.general.ReplicationFeatureFlagReader; @@ -38,11 +37,10 @@ public ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient) { + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, /* we don't care about the onReplicationRunning callback here */ () -> {}, false); + messageEventPublishingHelper, /* we don't care about the onReplicationRunning callback here */ () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java index 1ed8c7eb0f9..aa68cacacae 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java @@ -81,8 +81,7 @@ public abstract ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient); + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper); /** * Hook up the DefaultReplicationWorker to a test harness with an insanely quick Source @@ -117,7 +116,7 @@ public void executeOneSync() throws InterruptedException { final var syncPersistence = mock(SyncPersistence.class); final var connectorConfigUpdater = mock(ConnectorConfigUpdater.class); final var metricReporter = new WorkerMetricReporter(new NotImplementedMetricClient(), "test-image:0.01"); - final var dstNamespaceMapper = new NamespacingMapper(NamespaceDefinitionType.DESTINATION, "", "", false); + final var dstNamespaceMapper = new NamespacingMapper(NamespaceDefinitionType.DESTINATION, "", ""); final var validator = new RecordSchemaValidator(Map.of( new AirbyteStreamNameNamespacePair("s1", null), CatalogHelpers.fieldsToJsonSchema(io.airbyte.protocol.models.Field.of("data", JsonSchemaType.STRING)))); @@ -172,8 +171,7 @@ public void executeOneSync() throws InterruptedException { heartbeatTimeoutChaperone, new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, - replicationAirbyteMessageEventPublishingHelper, - featureFlagClient); + replicationAirbyteMessageEventPublishingHelper); final AtomicReference output = new AtomicReference<>(); final Thread workerThread = new Thread(() -> { try { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java index 680be9a7deb..da263da7e0e 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java @@ -4,17 +4,23 @@ package io.airbyte.workers.helper; +import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.commons.temporal.exception.SizeLimitException; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.FailureReason.FailureType; import io.airbyte.config.Metadata; import io.airbyte.protocol.models.AirbyteErrorTraceMessage; import io.airbyte.protocol.models.AirbyteTraceMessage; +import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.helper.FailureHelper.ConnectorCommand; import io.airbyte.workers.test_utils.AirbyteMessageUtils; +import io.temporal.api.enums.v1.RetryState; +import io.temporal.failure.ActivityFailure; import java.util.List; import java.util.Map; import java.util.Set; @@ -205,4 +211,15 @@ void testUnknownOriginFailure() { assertEquals("An unknown failure occurred", failureReason.getExternalMessage()); } + @Test + void testExceptionChainContains() { + final Throwable t = + new ActivityFailure(1L, 2L, "act", "actId", RetryState.RETRY_STATE_NON_RETRYABLE_FAILURE, "id", + new RuntimeException("oops", + new SizeLimitException("oops too big"))); + assertTrue(FailureHelper.exceptionChainContains(t, ActivityFailure.class)); + assertTrue(FailureHelper.exceptionChainContains(t, SizeLimitException.class)); + assertFalse(FailureHelper.exceptionChainContains(t, WorkerException.class)); + } + } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java index 6c3661edf4c..71eb02d3e94 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java @@ -64,7 +64,7 @@ void setUp() { @Test void testSourceNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -90,7 +90,7 @@ void testSourceNamespace() { @Test void testEmptySourceNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); assertEquals(originalCatalog, CATALOG); @@ -117,7 +117,7 @@ void testEmptySourceNamespace() { @Test void testDestinationNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.DESTINATION, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.DESTINATION, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -143,8 +143,7 @@ void testCustomFormatWithVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, "${SOURCE_NAMESPACE}_suffix", OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final String expectedNamespace = INPUT_NAMESPACE + "_suffix"; final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); @@ -172,8 +171,7 @@ void testCustomFormatWithoutVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, NAMESPACE_FORMAT, OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final String expectedNamespace = NAMESPACE_FORMAT; final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); @@ -201,8 +199,7 @@ void testEmptyCustomFormatWithVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, "${SOURCE_NAMESPACE}", OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); assertEquals(originalCatalog, CATALOG); @@ -232,8 +229,7 @@ void testMapStateMessage() { NamespaceDefinitionType.CUSTOMFORMAT, NAMESPACE_FORMAT, OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - true); + destinationToSourceNamespaceAndStreamName); final AirbyteMessage originalMessage = Jsons.clone(stateMessage); final AirbyteMessage expectedMessage = Jsons.clone(stateMessage); @@ -253,7 +249,7 @@ void testMapStateMessage() { @Test void testEmptyPrefix() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, null, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, null, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -280,7 +276,7 @@ void testEmptyPrefix() { @Test void testRevertMapStateMessage() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, true); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final AirbyteMessage originalMessage = Jsons.clone(stateMessage); originalMessage.getState().getStream().getStreamDescriptor().withNamespace(DESTINATION_NAMESPACE); @@ -296,7 +292,6 @@ void testRevertMapStateMessage() { expectedMessage.getState().getStream().getStreamDescriptor().withName(STREAM_NAME); assertEquals(expectedMessage, actualMessage); - } } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java index 7e42eda6a7e..daaa3478b09 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java @@ -123,7 +123,6 @@ void testEnvMapSet() throws IOException, WorkerException, InterruptedException { final WorkerConfigs workerConfigs = spy(new WorkerConfigs(new EnvConfigs())); when(workerConfigs.getEnvMap()).thenReturn(Map.of("ENV_VAR_1", "ENV_VALUE_1")); - when(workerConfigs.getEnvMap()).thenReturn(Map.of("ENV_VAR_1", "ENV_VALUE_1")); final DockerProcessFactory processFactory = new DockerProcessFactory( diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java index 01283fbee27..1c25cec4e03 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java @@ -16,6 +16,10 @@ public final class AuthRoleConstants { public static final String NONE = "NONE"; public static final String READER = "READER"; + public static final String ORGANIZATION_ADMIN = "ORGANIZATION_ADMIN"; + public static final String ORGANIZATION_EDITOR = "ORGANIZATION_EDITOR"; + public static final String ORGANIZATION_READER = "ORGANIZATION_READER"; + private AuthRoleConstants() {} } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java new file mode 100644 index 00000000000..068ebdef8fd --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This enum describes the Organization auth levels for a given resource. A user will have + * organization leveled auth role and workspace leveled auth roles. See AuthRole.java for more + * information. + */ +public enum OrganizationAuthRole { + + ORGANIZATION_ADMIN(400, AuthRoleConstants.ORGANIZATION_ADMIN), + ORGANIZATION_EDITOR(300, AuthRoleConstants.ORGANIZATION_EDITOR), + ORGANIZATION_READER(200, AuthRoleConstants.ORGANIZATION_READER), + NONE(0, AuthRoleConstants.NONE); + + private final int authority; + private final String label; + + OrganizationAuthRole(final int authority, final String label) { + this.authority = authority; + this.label = label; + } + + public int getAuthority() { + return authority; + } + + public String getLabel() { + return label; + } + + /** + * Builds the set of roles based on the provided {@link OrganizationAuthRole} value. + *

+ * The generated set of auth roles contains the provided {@link OrganizationAuthRole} (if not + * {@code null}) and any other authentication roles with a lesser {@link #getAuthority()} value. + *

+ * + * @param authRole An {@link OrganizationAuthRole} (may be {@code null}). + * @return The set of {@link OrganizationAuthRole} labels based on the provided + * {@link OrganizationAuthRole}. + */ + public static Set buildOrganizationAuthRolesSet(final OrganizationAuthRole authRole) { + final Set authRoles = new HashSet<>(); + + if (authRole != null) { + authRoles.add(authRole); + authRoles.addAll(Stream.of(values()) + .filter(role -> !NONE.equals(role)) + .filter(role -> role.getAuthority() < authRole.getAuthority()) + .collect(Collectors.toSet())); + } + + // Sort final set by descending authority order + return authRoles.stream() + .sorted(Comparator.comparingInt(OrganizationAuthRole::getAuthority)) + .map(role -> role.getLabel()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + +} diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index da0959e1a10..eb9a39718bf 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -812,4 +812,14 @@ enum SecretPersistenceType { AWS_SECRET_MANAGER } + /** + * The configured Airbyte edition for the instance. By default, an Airbyte instance is configured as + * Community edition. If configured as Pro edition, the instance will perform a license check and + * activate additional features if valid. + */ + enum AirbyteEdition { + COMMUNITY, + PRO + } + } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java index 229c149df61..a9583e6b932 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java @@ -13,6 +13,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.Collections; @@ -84,6 +85,7 @@ public static ActorDefinitionVersion toActorDefinitionVersion(@Nullable final Co .withDocumentationUrl(def.getDocumentationUrl()) .withProtocolVersion(getProtocolVersion(def.getSpec())) .withReleaseDate(def.getReleaseDate()) + .withSupportLevel(def.getSupportLevel() == null ? SupportLevel.NONE : def.getSupportLevel()) .withReleaseStage(def.getReleaseStage()) .withSuggestedStreams(def.getSuggestedStreams()); } @@ -108,6 +110,7 @@ public static ActorDefinitionVersion toActorDefinitionVersion(@Nullable final Co .withProtocolVersion(getProtocolVersion(def.getSpec())) .withReleaseDate(def.getReleaseDate()) .withReleaseStage(def.getReleaseStage()) + .withSupportLevel(def.getSupportLevel() == null ? SupportLevel.NONE : def.getSupportLevel()) .withNormalizationConfig(def.getNormalizationConfig()) .withSupportsDbt(def.getSupportsDbt()); } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index c17a51eabe2..0bc9e03694a 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -183,17 +183,25 @@ public void deleteLogs(final WorkerEnvironment workerEnvironment, final LogConfi * * @param workerEnvironment environment of worker. * @param logConfigs configuration for logs - * @param path log path + * @param path log path, if path is null, it will clear the JobMdc instead */ public void setJobMdc(final WorkerEnvironment workerEnvironment, final LogConfigs logConfigs, final Path path) { if (shouldUseLocalLogs(workerEnvironment)) { LOGGER.debug("Setting docker job mdc"); - final String resolvedPath = path.resolve(LogClientSingleton.LOG_FILENAME).toString(); - MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, resolvedPath); + if (path != null) { + final String resolvedPath = path.resolve(LogClientSingleton.LOG_FILENAME).toString(); + MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, resolvedPath); + } else { + MDC.remove(LogClientSingleton.JOB_LOG_PATH_MDC_KEY); + } } else { LOGGER.debug("Setting kube job mdc"); createCloudClientIfNull(logConfigs); - MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + if (path != null) { + MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } else { + MDC.remove(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY); + } } } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java deleted file mode 100644 index b926f123705..00000000000 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.helpers; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.util.ClassUtil; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * This is a convenience class for the conversion of a list of source/destination definitions from - * human-friendly yaml to processing friendly formats i.e. Java models or JSON. As this class - * performs validation, it is recommended to use this class to deal with plain lists. An example of - * such lists are Airbyte's master definition lists, which can be seen in the resources folder of - * the airbyte-config/seed module. - * - * In addition to usual deserialization validations, we check: 1) The given list contains no - * duplicate names. 2) The given list contains no duplicate ids. - * - * Methods in these class throw Runtime exceptions upon validation failure. - */ -@SuppressWarnings("PMD.ShortVariable") -public class YamlListToStandardDefinitions { - - private static final Map CLASS_NAME_TO_ID_NAME = Map.ofEntries( - new SimpleImmutableEntry<>(StandardDestinationDefinition.class.getCanonicalName(), "destinationDefinitionId"), - new SimpleImmutableEntry<>(StandardSourceDefinition.class.getCanonicalName(), "sourceDefinitionId")); - - public static List toStandardSourceDefinitions(final String yamlStr) { - return verifyAndConvertToModelList(StandardSourceDefinition.class, yamlStr); - } - - public static List toStandardDestinationDefinitions(final String yamlStr) { - return verifyAndConvertToModelList(StandardDestinationDefinition.class, yamlStr); - } - - static JsonNode verifyAndConvertToJsonNode(final String idName, final String yamlStr) { - final var jsonNode = Yamls.deserialize(yamlStr); - checkYamlIsPresentWithNoDuplicates(jsonNode, idName); - return jsonNode; - } - - @VisibleForTesting - static List verifyAndConvertToModelList(final Class klass, final String yamlStr) { - final var jsonNode = Yamls.deserialize(yamlStr); - final var idName = CLASS_NAME_TO_ID_NAME.get(klass.getCanonicalName()); - checkYamlIsPresentWithNoDuplicates(jsonNode, idName); - return toStandardXDefinitions(jsonNode.elements(), klass); - } - - private static void checkYamlIsPresentWithNoDuplicates(final JsonNode deserialize, final String idName) { - final var presentDestList = !deserialize.elements().equals(ClassUtil.emptyIterator()); - Preconditions.checkState(presentDestList, "Definition list is empty"); - checkNoDuplicateNames(deserialize.elements()); - checkNoDuplicateIds(deserialize.elements(), idName); - } - - private static void checkNoDuplicateNames(final Iterator iter) { - final var names = new HashSet(); - while (iter.hasNext()) { - final var element = Jsons.clone(iter.next()); - final var name = element.get("name").asText(); - if (names.contains(name)) { - throw new IllegalArgumentException("Multiple records have the name: " + name); - } - names.add(name); - } - } - - private static void checkNoDuplicateIds(final Iterator fileIterator, final String idName) { - final var ids = new HashSet(); - while (fileIterator.hasNext()) { - final var element = Jsons.clone(fileIterator.next()); - final var id = element.get(idName).asText(); - if (ids.contains(id)) { - throw new IllegalArgumentException("Multiple records have the id: " + id); - } - ids.add(id); - } - } - - private static List toStandardXDefinitions(final Iterator iter, final Class c) { - final Iterable iterable = () -> iter; - final var defList = new ArrayList(); - for (final JsonNode n : iterable) { - final var def = Jsons.object(n, c); - defList.add(def); - } - return defList; - } - -} diff --git a/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml b/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml index 933b83ecf04..22f55c4f2f3 100644 --- a/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml @@ -22,7 +22,12 @@ properties: type: string documentationUrl: type: string + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml index b43dbf31cc5..26b1f788e7f 100644 --- a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml @@ -42,7 +42,12 @@ properties: description: whether this is a custom connector definition type: boolean default: false + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml index 9b27f8ab0bf..b44f6c45c77 100644 --- a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml @@ -49,7 +49,12 @@ properties: description: whether this is a custom connector definition type: boolean default: false + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml b/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml index 3def0a0783d..a4647f78b0e 100644 --- a/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml +++ b/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml @@ -20,3 +20,7 @@ properties: $ref: NotificationItem.yaml sendOnConnectionUpdateActionRequired: $ref: NotificationItem.yaml + sendOnBreakingChangeWarning: + $ref: NotificationItem.yaml + sendOnBreakingChangeSyncsDisabled: + $ref: NotificationItem.yaml diff --git a/airbyte-config/config-models/src/main/resources/types/StreamSyncStats.yaml b/airbyte-config/config-models/src/main/resources/types/StreamSyncStats.yaml index 7fc3e12d29a..20e14065d37 100644 --- a/airbyte-config/config-models/src/main/resources/types/StreamSyncStats.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StreamSyncStats.yaml @@ -16,3 +16,6 @@ properties: type: string stats: "$ref": SyncStats.yaml + wasBackfilled: + type: boolean + description: Indicates whether the stream state was cleared before the sync diff --git a/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml b/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml new file mode 100644 index 00000000000..0afe1d03e83 --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml @@ -0,0 +1,10 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/SupportLevel.yaml +title: SupportLevel +description: enum that describes a connector's support level +type: string +enum: + - community + - certified + - none diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java index 7f6627c7d3b..a63687a5189 100644 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java +++ b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java @@ -25,6 +25,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.Collections; @@ -72,6 +73,7 @@ void testConvertRegistrySourceToInternalTypes() { .withTombstone(false) .withPublic(true) .withCustom(false) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion("doesnt matter") @@ -96,6 +98,7 @@ void testConvertRegistrySourceToInternalTypes() { .withDockerImageTag(DOCKER_TAG) .withSpec(SPEC) .withDocumentationUrl(DOCS_URL) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion(PROTOCOL_VERSION) @@ -107,6 +110,32 @@ void testConvertRegistrySourceToInternalTypes() { assertEquals(expectedBreakingChanges, ConnectorRegistryConverters.toActorDefinitionBreakingChanges(registrySourceDef)); } + @Test + void testConvertRegistrySourceDefaults() { + final SuggestedStreams suggestedStreams = new SuggestedStreams().withStreams(List.of("stream1", "stream2")); + final ConnectorRegistrySourceDefinition registrySourceDef = new ConnectorRegistrySourceDefinition() + .withSourceDefinitionId(DEF_ID) + .withName(CONNECTOR_NAME) + .withDockerRepository(DOCKER_REPOSITORY) + .withDockerImageTag(DOCKER_TAG) + .withDocumentationUrl(DOCS_URL) + .withSpec(SPEC) + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) + .withReleaseDate(RELEASE_DATE) + .withProtocolVersion("doesnt matter") + .withAllowedHosts(ALLOWED_HOSTS) + .withResourceRequirements(RESOURCE_REQUIREMENTS) + .withSuggestedStreams(suggestedStreams) + .withMaxSecondsBetweenMessages(10L) + .withReleases(new ConnectorReleases().withBreakingChanges(registryBreakingChanges)); + + final ActorDefinitionVersion convertedAdv = ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDef); + assertEquals(SupportLevel.NONE, convertedAdv.getSupportLevel()); + } + @Test void testConvertRegistryDestinationToInternalTypes() { final NormalizationDestinationDefinitionConfig normalizationConfig = new NormalizationDestinationDefinitionConfig() @@ -124,9 +153,10 @@ void testConvertRegistryDestinationToInternalTypes() { .withTombstone(false) .withPublic(true) .withCustom(false) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) - .withProtocolVersion("doesnt matter") + .withProtocolVersion(PROTOCOL_VERSION) .withAllowedHosts(ALLOWED_HOSTS) .withResourceRequirements(RESOURCE_REQUIREMENTS) .withNormalizationConfig(normalizationConfig) @@ -147,6 +177,7 @@ void testConvertRegistryDestinationToInternalTypes() { .withDockerImageTag(DOCKER_TAG) .withSpec(SPEC) .withDocumentationUrl(DOCS_URL) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion(PROTOCOL_VERSION) @@ -159,6 +190,30 @@ void testConvertRegistryDestinationToInternalTypes() { assertEquals(expectedBreakingChanges, ConnectorRegistryConverters.toActorDefinitionBreakingChanges(registryDestinationDef)); } + @Test + void testConvertRegistryDestinationDefaults() { + final ConnectorRegistryDestinationDefinition registryDestinationDef = new ConnectorRegistryDestinationDefinition() + .withDestinationDefinitionId(DEF_ID) + .withName(CONNECTOR_NAME) + .withDockerRepository(DOCKER_REPOSITORY) + .withDockerImageTag(DOCKER_TAG) + .withDocumentationUrl(DOCS_URL) + .withSpec(SPEC) + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) + .withReleaseDate(RELEASE_DATE) + .withProtocolVersion(PROTOCOL_VERSION) + .withAllowedHosts(ALLOWED_HOSTS) + .withResourceRequirements(RESOURCE_REQUIREMENTS) + .withSupportsDbt(true) + .withReleases(new ConnectorReleases().withBreakingChanges(registryBreakingChanges)); + + final ActorDefinitionVersion convertedAdv = ConnectorRegistryConverters.toActorDefinitionVersion(registryDestinationDef); + assertEquals(SupportLevel.NONE, convertedAdv.getSupportLevel()); + } + @Test void testParseSourceDefinitionWithNoBreakingChangesReturnsEmptyList() { ConnectorRegistrySourceDefinition registrySourceDef = new ConnectorRegistrySourceDefinition(); diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java deleted file mode 100644 index 56912b5774e..00000000000 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.helpers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.config.StandardDestinationDefinition; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class YamlListToStandardDefinitionsTest { - - private static final String DESTINATION_DEFINITION_ID = "- destinationDefinitionId: a625d593-bba5-4a1c-a53d-2d246268a816\n"; - private static final String DESTINATION_NAME = " name: Local JSON\n"; - private static final String DOCKER_REPO = " dockerRepository: airbyte/destination-local-json\n"; - private static final String DOCKER_IMAGE_TAG = " dockerImageTag: 0.1.4\n"; - private static final String GOOD_DES_DEF_YAML = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json"; - private static final String DUPLICATE_ID = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json" - + DESTINATION_DEFINITION_ID - + " name: JSON 2\n" - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json"; - private static final String DUPLICATE_NAME = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json\n" - + "- destinationDefinitionId: 8be1cf83-fde1-477f-a4ad-318d23c9f3c6\n" - + DESTINATION_NAME - + " dockerRepository: airbyte/destination-csv\n" - + " dockerImageTag: 0.1.8\n" - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-csv"; - private static final String BAD_DATA = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + " dockerImageTag: 0.1.8\n" - + " documentationUrl"; - - @Nested - @DisplayName("vertifyAndConvertToJsonNode") - class VerifyAndConvertToJsonNode { - - private static final String ID_NAME = "destinationDefinitionId"; - - private final ObjectMapper mapper = MoreMappers.initMapper(); - - @Test - @DisplayName("should correctly read yaml file") - void correctlyReadTest() throws JsonProcessingException { - final var jsonDefs = YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, GOOD_DES_DEF_YAML); - final var defList = mapper.treeToValue(jsonDefs, StandardDestinationDefinition[].class); - assertEquals(1, defList.length); - assertEquals("Local JSON", defList[0].getName()); - } - - @Test - @DisplayName("should error out on duplicate id") - void duplicateIdTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, DUPLICATE_ID)); - } - - @Test - @DisplayName("should error out on duplicate name") - void duplicateNameTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, DUPLICATE_NAME)); - } - - @Test - @DisplayName("should error out on empty file") - void emptyFileTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, "")); - } - - @Test - @DisplayName("should error out on bad data") - void badDataTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, BAD_DATA)); - } - - } - - @Nested - @DisplayName("verifyAndConvertToModelList") - class VerifyAndConvertToModelList { - - @Test - @DisplayName("should correctly read yaml file") - void correctlyReadTest() { - final var defs = YamlListToStandardDefinitions - .verifyAndConvertToModelList(StandardDestinationDefinition.class, GOOD_DES_DEF_YAML); - assertEquals(1, defs.size()); - assertEquals("Local JSON", defs.get(0).getName()); - } - - @Test - @DisplayName("should error out on duplicate id") - void duplicateIdTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, DUPLICATE_ID)); - } - - @Test - @DisplayName("should error out on duplicate name") - void duplicateNameTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, DUPLICATE_NAME)); - } - - @Test - @DisplayName("should error out on empty file") - void emptyFileTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, "")); - } - - @Test - @DisplayName("should error out on bad data") - void badDataTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, BAD_DATA)); - } - - } - -} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java index e21fafa21e3..326003f160e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java @@ -33,6 +33,15 @@ @Singleton public class ActorDefinitionVersionHelper { + /** + * A wrapper class for returning the actor definition version and whether an override was applied. + * + * @param actorDefinitionVersion - actor definition version to use + * @param isOverrideApplied - true if the version is the result of an override being applied, + * otherwise false + */ + public record ActorDefinitionVersionWithOverrideStatus(ActorDefinitionVersion actorDefinitionVersion, boolean isOverrideApplied) {} + private static final Logger LOGGER = LoggerFactory.getLogger(ActorDefinitionVersionHelper.class); private final ConfigRepository configRepository; @@ -90,16 +99,16 @@ private ActorDefinitionVersion getDefaultDestinationVersion(final StandardDestin } /** - * Get the actor definition version to use for a source. + * Get the actor definition version to use for a source, and whether an override was applied. * * @param sourceDefinition source definition * @param workspaceId workspace id * @param actorId source id - * @return actor definition version + * @return actor definition version with override status */ - public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition sourceDefinition, - final UUID workspaceId, - @Nullable final UUID actorId) + public ActorDefinitionVersionWithOverrideStatus getSourceVersionWithOverrideStatus(final StandardSourceDefinition sourceDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion defaultVersion = getDefaultSourceVersion(sourceDefinition, workspaceId, actorId); @@ -110,7 +119,22 @@ public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition so actorId, defaultVersion); - return versionOverride.orElse(defaultVersion); + return new ActorDefinitionVersionWithOverrideStatus(versionOverride.orElse(defaultVersion), versionOverride.isPresent()); + } + + /** + * Get the actor definition version to use for a source. + * + * @param sourceDefinition source definition + * @param workspaceId workspace id + * @param actorId source id + * @return actor definition version + */ + public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition sourceDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) + throws ConfigNotFoundException, IOException, JsonValidationException { + return getSourceVersionWithOverrideStatus(sourceDefinition, workspaceId, actorId).actorDefinitionVersion(); } /** @@ -126,16 +150,16 @@ public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition so } /** - * Get the actor definition version to use for a destination. + * Get the actor definition version to use for a destination, and whether an override was applied. * * @param destinationDefinition destination definition * @param workspaceId workspace id * @param actorId destination id - * @return actor definition version + * @return actor definition version with override status */ - public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDefinition destinationDefinition, - final UUID workspaceId, - @Nullable final UUID actorId) + public ActorDefinitionVersionWithOverrideStatus getDestinationVersionWithOverrideStatus(final StandardDestinationDefinition destinationDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion defaultVersion = getDefaultDestinationVersion(destinationDefinition, workspaceId, actorId); @@ -146,7 +170,22 @@ public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDef actorId, defaultVersion); - return versionOverride.orElse(defaultVersion); + return new ActorDefinitionVersionWithOverrideStatus(versionOverride.orElse(defaultVersion), versionOverride.isPresent()); + } + + /** + * Get the actor definition version to use for a destination. + * + * @param destinationDefinition destination definition + * @param workspaceId workspace id + * @param actorId destination id + * @return actor definition version + */ + public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDefinition destinationDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) + throws ConfigNotFoundException, IOException, JsonValidationException { + return getDestinationVersionWithOverrideStatus(destinationDefinition, workspaceId, actorId).actorDefinitionVersion(); } /** diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index c4f052c5524..2b5f837a313 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -76,6 +76,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; import io.airbyte.db.instance.configs.jooq.generated.enums.ScopeType; import io.airbyte.db.instance.configs.jooq.generated.enums.StatusType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.enums.SupportState; import io.airbyte.db.instance.configs.jooq.generated.tables.records.ActorDefinitionWorkspaceGrantRecord; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; @@ -95,6 +96,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -201,6 +203,20 @@ public record ResourcesByOrganizationQueryPaginated( } + /** + * Query object for paginated querying of resource for a user. + * + * @param userId user to fetch resources for + * @param includeDeleted include tombstoned resources + * @param pageSize limit + * @param rowOffset offset + */ + public record ResourcesByUserQueryPaginated( + @Nonnull UUID userId, + boolean includeDeleted, + int pageSize, + int rowOffset) {} + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigRepository.class); private static final String OPERATION_IDS_AGG_FIELD = "operation_ids_agg"; private static final String OPERATION_IDS_AGG_DELIMITER = ","; @@ -223,9 +239,9 @@ public ConfigRepository(final Database database, final Supplier heartbeatM } @VisibleForTesting - ConfigRepository(final Database database, - final StandardSyncPersistence standardSyncPersistence, - final Supplier heartbeatMaxSecondBetweenMessageSupplier) { + public ConfigRepository(final Database database, + final StandardSyncPersistence standardSyncPersistence, + final Supplier heartbeatMaxSecondBetweenMessageSupplier) { this.database = new ExceptionWrappingDatabase(database); this.standardSyncPersistence = standardSyncPersistence; this.heartbeatMaxSecondBetweenMessageSupplier = heartbeatMaxSecondBetweenMessageSupplier; @@ -533,7 +549,8 @@ public void writeStandardWorkspaceNoSecrets(final StandardWorkspace workspace) t * @throws IOException - you never know when you IO */ public void setFeedback(final UUID workspaceId) throws IOException { - database.query(ctx -> ctx.update(WORKSPACE).set(WORKSPACE.FEEDBACK_COMPLETE, true).where(WORKSPACE.ID.eq(workspaceId)).execute()); + database.query(ctx -> ctx.update(WORKSPACE).set(WORKSPACE.FEEDBACK_COMPLETE, true).set(WORKSPACE.UPDATED_AT, OffsetDateTime.now()) + .where(WORKSPACE.ID.eq(workspaceId)).execute()); } /** @@ -756,48 +773,6 @@ public void updateStandardSourceDefinition(final StandardSourceDefinition source }); } - /** - * Write a StandardSourceDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardSourceDefinition. - * - * @param sourceDefinition standard source definition - * @param actorDefinitionVersion actor definition version - * @param breakingChangesForDefinition - list of breaking changes for the definition - * @throws IOException - you never know when you IO - */ - public void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition) - throws IOException { - database.transaction(ctx -> { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); - return null; - }); - } - - /** - * Write a StandardSourceDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardSourceDefinition. Assumes the - * definition has no breaking changes. - * - * @param sourceDefinition standard source definition - * @param actorDefinitionVersion actor definition version - * @throws IOException - you never know when you IO - */ - public void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion) - throws IOException { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, List.of()); - } - - private void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - ConfigWriter.writeStandardSourceDefinition(Collections.singletonList(sourceDefinition), ctx); - setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); - } - /** * Update the docker image tag for multiple actor definitions at once. * @@ -809,33 +784,13 @@ public int updateActorDefinitionsDockerImageTag(final List actorDefinition return database.transaction(ctx -> ConfigWriter.writeSourceDefinitionImageTag(actorDefinitionIds, targetImageTag, ctx)); } - /** - * Write custom source definition and its default version. - * - * @param sourceDefinition source definition - * @param defaultVersion default version - * @param scopeId scope id - * @param scopeType enum which defines if the scopeId is a workspace or organization id - * @throws IOException - you never know when you IO - */ - public void writeCustomSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion defaultVersion, - final UUID scopeId, - final io.airbyte.config.ScopeType scopeType) - throws IOException { - database.transaction(ctx -> { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, defaultVersion, List.of(), ctx); - writeActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); - return null; - }); - } - private void updateDeclarativeActorDefinition(final ActorDefinitionConfigInjection configInjection, final ConnectorSpecification spec, final DSLContext ctx) { // We are updating the same version since connector builder projects have a different concept of // versioning. ctx.update(ACTOR_DEFINITION_VERSION) + .set(ACTOR_DEFINITION_VERSION.UPDATED_AT, OffsetDateTime.now()) .set(ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(spec))) .where(ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID.eq(configInjection.getActorDefinitionId())) .execute(); @@ -985,44 +940,96 @@ public void updateStandardDestinationDefinition(final StandardDestinationDefinit } /** - * Write a StandardDestinationDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardDestinationDefinition. + * Write metadata for a destination connector. Writes global metadata (destination definition) and + * versioned metadata (info for actor definition version to set as default). Sets the new version as + * the default version and updates actors accordingly, based on whether the upgrade will be breaking + * or not. * * @param destinationDefinition standard destination definition * @param actorDefinitionVersion actor definition version * @param breakingChangesForDefinition - list of breaking changes for the definition * @throws IOException - you never know when you IO */ - public void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition) + public void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition) throws IOException { database.transaction(ctx -> { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); + writeConnectorMetadata(destinationDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); return null; }); } /** - * Write a StandardDestinationDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardDestinationDefinition. - * Assumes the definition has no breaking changes. + * Write metadata for a destination connector. Writes global metadata (destination definition) and + * versioned metadata (info for actor definition version to set as default). Sets the new version as + * the default version and updates actors accordingly, based on whether the upgrade will be breaking + * or not. Usage of this version of the method assumes no new breaking changes need to be persisted + * for the definition. * * @param destinationDefinition standard destination definition * @param actorDefinitionVersion actor definition version * @throws IOException - you never know when you IO */ - public void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion) + public void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion) throws IOException { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, List.of()); + writeConnectorMetadata(destinationDefinition, actorDefinitionVersion, List.of()); } - private void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { + private void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { ConfigWriter.writeStandardDestinationDefinition(Collections.singletonList(destinationDefinition), ctx); + writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); + setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); + } + + /** + * Write metadata for a source connector. Writes global metadata (source definition, breaking + * changes) and versioned metadata (info for actor definition version to set as default). Sets the + * new version as the default version and updates actors accordingly, based on whether the upgrade + * will be breaking or not. + * + * @param sourceDefinition standard source definition + * @param actorDefinitionVersion actor definition version, containing tag to set as default + * @param breakingChangesForDefinition - list of breaking changes for the definition + * @throws IOException - you never know when you IO + */ + public void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition) + throws IOException { + database.transaction(ctx -> { + writeConnectorMetadata(sourceDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); + return null; + }); + } + + /** + * Write metadata for a source connector. Writes global metadata (source definition) and versioned + * metadata (info for actor definition version to set as default). Sets the new version as the + * default version and updates actors accordingly, based on whether the upgrade will be breaking or + * not. Usage of this version of the method assumes no new breaking changes need to be persisted for + * the definition. + * + * @param sourceDefinition standard source definition + * @param actorDefinitionVersion actor definition version + * @throws IOException - you never know when you IO + */ + public void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion) + throws IOException { + writeConnectorMetadata(sourceDefinition, actorDefinitionVersion, List.of()); + } + + private void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { + ConfigWriter.writeStandardSourceDefinition(Collections.singletonList(sourceDefinition), ctx); + writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); } @@ -1072,38 +1079,64 @@ private void setActorDefinitionVersionAsDefaultVersion(final ActorDefinitionVers private void updateDefaultVersionIdForActorsOnVersion(final UUID previousDefaultVersionId, final UUID newDefaultVersionId, final DSLContext ctx) { ctx.update(ACTOR) - .set(Tables.ACTOR.DEFAULT_VERSION_ID, newDefaultVersionId) - .where(Tables.ACTOR.DEFAULT_VERSION_ID.eq(previousDefaultVersionId)) + .set(ACTOR.UPDATED_AT, OffsetDateTime.now()) + .set(ACTOR.DEFAULT_VERSION_ID, newDefaultVersionId) + .where(ACTOR.DEFAULT_VERSION_ID.eq(previousDefaultVersionId)) .execute(); } private void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, final UUID versionId, final DSLContext ctx) { ctx.update(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.UPDATED_AT, OffsetDateTime.now()) .set(ACTOR_DEFINITION.DEFAULT_VERSION_ID, versionId) .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId)) .execute(); } /** - * Write custom destination definition and its default version. + * Write metadata for a custom destination: global metadata (destination definition) and versioned + * metadata (actor definition version for the version to use). * * @param destinationDefinition destination definition + * @param defaultVersion default actor definition version * @param scopeId workspace or organization id * @param scopeType enum of workpsace or organization * @throws IOException - you never know when you IO */ - public void writeCustomDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion defaultVersion, - final UUID scopeId, - final io.airbyte.config.ScopeType scopeType) + public void writeCustomConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion defaultVersion, + final UUID scopeId, + final io.airbyte.config.ScopeType scopeType) throws IOException { database.transaction(ctx -> { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, defaultVersion, List.of(), ctx); + writeConnectorMetadata(destinationDefinition, defaultVersion, List.of(), ctx); writeActorDefinitionWorkspaceGrant(destinationDefinition.getDestinationDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); return null; }); } + /** + * Write metadata for a custom source: global metadata (source definition) and versioned metadata + * (actor definition version for the version to use). + * + * @param sourceDefinition source definition + * @param defaultVersion default actor definition version + * @param scopeId scope id + * @param scopeType enum which defines if the scopeId is a workspace or organization id + * @throws IOException - you never know when you IO + */ + public void writeCustomConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion defaultVersion, + final UUID scopeId, + final io.airbyte.config.ScopeType scopeType) + throws IOException { + database.transaction(ctx -> { + writeConnectorMetadata(sourceDefinition, defaultVersion, List.of(), ctx); + writeActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); + return null; + }); + } + /** * Delete connection. * @@ -1563,20 +1596,23 @@ public List listWorkspacesDestinationConnections(final Re } /** - * List workspace IDs with most recently running jobs within a given time window (in hours). + * List active workspace IDs with most recently running jobs within a given time window (in hours). * * @param timeWindowInHours - integer, e.g. 24, 48, etc * @return list of workspace IDs * @throws IOException - failed to query data */ - public List listWorkspacesByMostRecentlyRunningJobs(final int timeWindowInHours) throws IOException { + public List listActiveWorkspacesByMostRecentlyRunningJobs(final int timeWindowInHours) throws IOException { final Result> records = database.query(ctx -> ctx.selectDistinct(ACTOR.WORKSPACE_ID) .from(ACTOR) + .join(WORKSPACE) + .on(ACTOR.WORKSPACE_ID.eq(WORKSPACE.ID)) .join(CONNECTION) .on(CONNECTION.SOURCE_ID.eq(ACTOR.ID)) .join(JOBS) .on(CONNECTION.ID.cast(VARCHAR(255)).eq(JOBS.SCOPE)) .where(JOBS.UPDATED_AT.greaterOrEqual(OffsetDateTime.now().minusHours(timeWindowInHours))) + .and(WORKSPACE.TOMBSTONE.isFalse()) .fetch()); return records.stream().map(record -> record.get(ACTOR.WORKSPACE_ID)).collect(Collectors.toList()); } @@ -1597,6 +1633,38 @@ public List listSourcesForDefinition(final UUID definitionId) return result.stream().map(DbConverter::buildSourceConnection).collect(Collectors.toList()); } + /** + * Returns all active sources whose default_version_id is in a given list of version IDs. + * + * @param actorDefinitionVersionIds - list of actor definition version ids + * @return list of SourceConnections + * @throws IOException - you never know when you IO + */ + public List listSourcesWithVersionIds(final List actorDefinitionVersionIds) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.source)) + .and(ACTOR.DEFAULT_VERSION_ID.in(actorDefinitionVersionIds)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildSourceConnection).toList(); + } + + /** + * Returns all active destinations whose default_version_id is in a given list of version IDs. + * + * @param actorDefinitionVersionIds - list of actor definition version ids + * @return list of DestinationConnections + * @throws IOException - you never know when you IO + */ + public List listDestinationsWithVersionIds(final List actorDefinitionVersionIds) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.destination)) + .and(ACTOR.DEFAULT_VERSION_ID.in(actorDefinitionVersionIds)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildDestinationConnection).toList(); + } + /** * Returns all active destinations using a definition. * @@ -2060,6 +2128,7 @@ public void deleteStandardSyncOperation(final UUID standardSyncOperationId) thro ctx.deleteFrom(CONNECTION_OPERATION) .where(CONNECTION_OPERATION.OPERATION_ID.eq(standardSyncOperationId)).execute(); ctx.update(OPERATION) + .set(OPERATION.UPDATED_AT, OffsetDateTime.now()) .set(OPERATION.TOMBSTONE, true) .where(OPERATION.ID.eq(standardSyncOperationId)).execute(); return null; @@ -2369,79 +2438,66 @@ public ActorCatalog getActorCatalogById(final UUID actorCatalogId) } /** - * Store an Airbyte catalog in DB if it is not present already. - *

- * Checks in the config DB if the catalog is present already, if so returns it identifier. It is not - * present, it is inserted in DB with a new identifier and that identifier is returned. + * Store an Airbyte catalog in DB if it is not present already. Checks in the config DB if the + * catalog is present already, if so returns it identifier. If not present, it is inserted in DB + * with a new identifier and that identifier is returned. * - * @param airbyteCatalog An Airbyte catalog to cache + * @param airbyteCatalog the catalog to be cached * @param context - db context + * @param timestamp - timestamp * @return the db identifier for the cached catalog. */ private UUID getOrInsertActorCatalog(final AirbyteCatalog airbyteCatalog, final DSLContext context, final OffsetDateTime timestamp) { - final HashFunction hashFunction = Hashing.murmur3_32_fixed(); - final String catalogHash = hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes( - Charsets.UTF_8)).toString(); - final Map catalogs = findCatalogByHash(catalogHash, context); - for (final Map.Entry entry : catalogs.entrySet()) { - if (entry.getValue().equals(airbyteCatalog)) { - return entry.getKey(); - } + final String canonicalCatalogHash = generateCanonicalHash(airbyteCatalog); + UUID catalogId = lookupCatalogId(canonicalCatalogHash, airbyteCatalog, context); + if (catalogId != null) { + return catalogId; } - final UUID catalogId = UUID.randomUUID(); - context.insertInto(ACTOR_CATALOG) - .set(ACTOR_CATALOG.ID, catalogId) - .set(ACTOR_CATALOG.CATALOG, JSONB.valueOf(Jsons.serialize(airbyteCatalog))) - .set(ACTOR_CATALOG.CATALOG_HASH, catalogHash) - .set(ACTOR_CATALOG.CREATED_AT, timestamp) - .set(ACTOR_CATALOG.MODIFIED_AT, timestamp).execute(); - return catalogId; + final String oldCatalogHash = generateOldHash(airbyteCatalog); + catalogId = lookupCatalogId(oldCatalogHash, airbyteCatalog, context); + if (catalogId != null) { + return catalogId; + } + + return insertCatalog(airbyteCatalog, canonicalCatalogHash, context, timestamp); } - /** - * This function will be used to gradually migrate the existing data in the database to use the - * canonical json serialization. It will first try to find the catalog using the canonical json - * serialization. If it fails, it will fallback to the old json serialization. - * - * @param airbyteCatalog the catalog to be cached - * @param context - db context - * @param timestamp - timestamp - * @return the db identifier for the cached catalog. - */ - private UUID getOrInsertCanonicalActorCatalog(final AirbyteCatalog airbyteCatalog, - final DSLContext context, - final OffsetDateTime timestamp) { + private String generateCanonicalHash(final AirbyteCatalog airbyteCatalog) { final HashFunction hashFunction = Hashing.murmur3_32_fixed(); - try { - final String catalogHash = hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog) + return hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog) .getBytes(Charsets.UTF_8)).toString(); - - final UUID catalogId = findAndReturnCatalogId(catalogHash, airbyteCatalog, context); - if (catalogId != null) { - return catalogId; - } } catch (final IOException e) { LOGGER.error("Failed to serialize AirbyteCatalog to canonical JSON", e); + return null; } + } - // Fallback to the old json when canonical json serialization failed - final String oldCatalogHash = hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); + private String generateOldHash(final AirbyteCatalog airbyteCatalog) { + final HashFunction hashFunction = Hashing.murmur3_32_fixed(); + return hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); + } - final UUID oldCatalogId = findAndReturnCatalogId(oldCatalogHash, airbyteCatalog, context); - if (oldCatalogId != null) { - return oldCatalogId; + private UUID lookupCatalogId(final String catalogHash, final AirbyteCatalog airbyteCatalog, final DSLContext context) { + if (catalogHash == null) { + return null; } + return findAndReturnCatalogId(catalogHash, airbyteCatalog, context); + } + private UUID insertCatalog(final AirbyteCatalog airbyteCatalog, + final String catalogHash, + final DSLContext context, + final OffsetDateTime timestamp) { final UUID catalogId = UUID.randomUUID(); context.insertInto(ACTOR_CATALOG) .set(ACTOR_CATALOG.ID, catalogId) .set(ACTOR_CATALOG.CATALOG, JSONB.valueOf(Jsons.serialize(airbyteCatalog))) - .set(ACTOR_CATALOG.CATALOG_HASH, oldCatalogHash) + .set(ACTOR_CATALOG.CATALOG_HASH, catalogHash) .set(ACTOR_CATALOG.CREATED_AT, timestamp) .set(ACTOR_CATALOG.MODIFIED_AT, timestamp).execute(); return catalogId; @@ -2600,39 +2656,6 @@ public UUID writeActorCatalogFetchEvent(final AirbyteCatalog catalog, }); } - /** - * This function will be used to gradually transition to reading and writing canonical schemas. - * Eventually, the writeActorCatalogFetchEvent function will be removed and this function will be - * renamed to writeActorCatalogFetchEvent. - * - * @param catalog - catalog that was fetched. - * @param actorId - actor the catalog was fetched by - * @param connectorVersion - version of the connector when catalog was fetched - * @param configurationHash - hash of the config of the connector when catalog was fetched - * @return The identifier (UUID) of the fetch event inserted in the database - * @throws IOException - error while interacting with db - */ - public UUID writeCanonicalActorCatalogFetchEvent(final AirbyteCatalog catalog, - final UUID actorId, - final String connectorVersion, - final String configurationHash) - throws IOException { - final OffsetDateTime timestamp = OffsetDateTime.now(); - final UUID fetchEventID = UUID.randomUUID(); - return database.transaction(ctx -> { - final UUID catalogId = getOrInsertCanonicalActorCatalog(catalog, ctx, timestamp); - ctx.insertInto(ACTOR_CATALOG_FETCH_EVENT) - .set(ACTOR_CATALOG_FETCH_EVENT.ID, fetchEventID) - .set(ACTOR_CATALOG_FETCH_EVENT.ACTOR_ID, actorId) - .set(ACTOR_CATALOG_FETCH_EVENT.ACTOR_CATALOG_ID, catalogId) - .set(ACTOR_CATALOG_FETCH_EVENT.CONFIG_HASH, configurationHash) - .set(ACTOR_CATALOG_FETCH_EVENT.ACTOR_VERSION, connectorVersion) - .set(ACTOR_CATALOG_FETCH_EVENT.MODIFIED_AT, timestamp) - .set(ACTOR_CATALOG_FETCH_EVENT.CREATED_AT, timestamp).execute(); - return catalogId; - }); - } - /** * Count connections in workspace. * @@ -2985,6 +3008,7 @@ public Stream getConnectorBuilderProjectsByWorkspace(@N */ public boolean deleteBuilderProject(final UUID builderProjectId) throws IOException { return database.transaction(ctx -> ctx.update(CONNECTOR_BUILDER_PROJECT).set(CONNECTOR_BUILDER_PROJECT.TOMBSTONE, true) + .set(CONNECTOR_BUILDER_PROJECT.UPDATED_AT, OffsetDateTime.now()) .where(CONNECTOR_BUILDER_PROJECT.ID.eq(builderProjectId)).execute()) > 0; } @@ -3051,6 +3075,7 @@ public void deleteBuilderProjectDraft(final UUID projectId) throws IOException { database.transaction(ctx -> { ctx.update(CONNECTOR_BUILDER_PROJECT) .setNull(CONNECTOR_BUILDER_PROJECT.MANIFEST_DRAFT) + .set(CONNECTOR_BUILDER_PROJECT.UPDATED_AT, OffsetDateTime.now()) .where(CONNECTOR_BUILDER_PROJECT.ID.eq(projectId)) .execute(); return null; @@ -3069,6 +3094,7 @@ public void deleteManifestDraftForActorDefinition(final UUID actorDefinitionId, database.transaction(ctx -> { ctx.update(CONNECTOR_BUILDER_PROJECT) .setNull(CONNECTOR_BUILDER_PROJECT.MANIFEST_DRAFT) + .set(CONNECTOR_BUILDER_PROJECT.UPDATED_AT, OffsetDateTime.now()) .where(CONNECTOR_BUILDER_PROJECT.ACTOR_DEFINITION_ID.eq(actorDefinitionId) .and(CONNECTOR_BUILDER_PROJECT.WORKSPACE_ID.eq(workspaceId))) .execute(); @@ -3102,6 +3128,7 @@ public void updateBuilderProjectAndActorDefinition(final UUID projectId, database.transaction(ctx -> { writeBuilderProjectDraft(projectId, workspaceId, name, manifestDraft, ctx); ctx.update(ACTOR_DEFINITION) + .set(ACTOR_DEFINITION.UPDATED_AT, OffsetDateTime.now()) .set(ACTOR_DEFINITION.NAME, name) .where(ACTOR_DEFINITION.ID.eq(actorDefinitionId).and(ACTOR_DEFINITION.PUBLIC.eq(false))) .execute(); @@ -3120,6 +3147,7 @@ public void assignActorDefinitionToConnectorBuilderProject(final UUID builderPro database.transaction(ctx -> { ctx.update(CONNECTOR_BUILDER_PROJECT) .set(CONNECTOR_BUILDER_PROJECT.ACTOR_DEFINITION_ID, actorDefinitionId) + .set(CONNECTOR_BUILDER_PROJECT.UPDATED_AT, OffsetDateTime.now()) .where(CONNECTOR_BUILDER_PROJECT.ID.eq(builderProjectId)) .execute(); return null; @@ -3488,6 +3516,7 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV final OffsetDateTime timestamp = OffsetDateTime.now(); // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); + ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) @@ -3498,6 +3527,9 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) + .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null + : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), + SupportLevel.class).orElseThrow()) .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), ReleaseStage.class).orElseThrow()) @@ -3524,6 +3556,7 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, Enums.toEnum(actorDefinitionVersion.getSupportState().value(), SupportState.class).orElseThrow()) .execute(); + return actorDefinitionVersion.withVersionId(versionId); } @@ -3615,13 +3648,15 @@ public List getActorDefinitionVersions(final List * already exist. * * @param breakingChanges - actor definition breaking changes to write + * @param ctx database context * @throws IOException - you never know when you io */ - public void writeActorDefinitionBreakingChanges(final List breakingChanges) throws IOException { + private void writeActorDefinitionBreakingChanges(final List breakingChanges, final DSLContext ctx) { final OffsetDateTime timestamp = OffsetDateTime.now(); - database.query(ctx -> ctx - .batch(breakingChanges.stream().map(breakingChange -> upsertBreakingChangeQuery(ctx, breakingChange, timestamp)).collect(Collectors.toList())) - .execute()); + final List upsertQueries = breakingChanges.stream() + .map(breakingChange -> upsertBreakingChangeQuery(ctx, breakingChange, timestamp)) + .collect(Collectors.toList()); + ctx.batch(upsertQueries).execute(); } /** @@ -3633,6 +3668,7 @@ public void writeActorDefinitionBreakingChanges(final List ctx.update(Tables.ACTOR) .set(Tables.ACTOR.DEFAULT_VERSION_ID, actorDefinitionVersionId) + .set(Tables.ACTOR.UPDATED_AT, OffsetDateTime.now()) .where(Tables.ACTOR.ID.eq(actorId)) .execute()); } @@ -3710,6 +3746,7 @@ public void setActorDefinitionVersionSupportStates(final List actorDefinit throws IOException { database.query(ctx -> ctx.update(Tables.ACTOR_DEFINITION_VERSION) .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, Enums.toEnum(supportState.value(), SupportState.class).orElseThrow()) + .set(Tables.ACTOR_DEFINITION_VERSION.UPDATED_AT, OffsetDateTime.now()) .where(Tables.ACTOR_DEFINITION_VERSION.ID.in(actorDefinitionVersionIds)) .execute()); } @@ -3756,4 +3793,51 @@ public List listBreakingChanges() throws IOExcept .collect(Collectors.toList())); } + /** + * This query retrieves successful sync jobs for connections that have been created in the past 7 + * days OR finds the first successful sync jobs for their corresponding connections. These results + * are used to mark these early syncs as free. + */ + private static final String EARLY_SYNC_JOB_QUERY = + // Find the first successful sync job ID for every connection. + // This will be used in a join below to check if a particular job is the connection's + // first successful sync + "WITH FirstSuccessfulJobIdByConnection AS (" + + " SELECT j2.scope, MIN(j2.id) AS min_job_id" + + " FROM jobs j2" + + " WHERE j2.status = 'succeeded' AND j2.config_type = 'sync'" + + " GROUP BY j2.scope" + + ")" + // Left join Jobs on Connection and the above MinJobIds, and only keep successful + // sync jobs that have an associated Connection ID + + " SELECT j.id, j.created_at, c.id, c.created_at AS connection_created_at, min_job_id" + + " FROM jobs j" + + " LEFT JOIN connection c ON c.id = UUID(j.scope)" + + " LEFT JOIN FirstSuccessfulJobIdByConnection min_j_ids ON j.id = min_j_ids.min_job_id" + + " WHERE j.status = 'succeeded'" + + " AND j.config_type = 'sync'" + + " AND c.id IS NOT NULL" + // Keep a job if it was created within 7 days of its connection's creation, + // OR if it was the first successful sync job of its connection + + " AND ((j.created_at < c.created_at + make_interval(days => ?))" + + " OR min_job_id IS NOT NULL)" + // Only consider jobs that were created in the last 30 days, to cut down the query size. + + " AND j.created_at > now() - make_interval(days => ?);"; + + public Set listEarlySyncJobs(final int freeUsageInterval, final int jobsFetchRange) + throws IOException { + return database.query(ctx -> getEarlySyncJobsFromResult(ctx.fetch( + EARLY_SYNC_JOB_QUERY, freeUsageInterval, jobsFetchRange))); + } + + private Set getEarlySyncJobsFromResult(final Result result) { + // Transform the result to a list of early sync job ids + // the rest of the fields are not used, we aim to keep the set small + final Set earlySyncJobs = new HashSet<>(); + for (final Record record : result) { + earlySyncJobs.add((Long) record.get("id")); + } + return earlySyncJobs; + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java index 3fafe705124..5069935665e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java @@ -61,6 +61,7 @@ import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.db.instance.configs.jooq.generated.enums.AutoPropagationStatus; import io.airbyte.db.instance.configs.jooq.generated.enums.NotificationType; @@ -472,6 +473,8 @@ public static ActorDefinitionVersion buildActorDefinitionVersion(final Record re .withDockerImageTag(record.get(ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG)) .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION_VERSION.SPEC).data(), ConnectorSpecification.class)) .withDocumentationUrl(record.get(ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL)) + .withSupportLevel(record.get(ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) == null ? null + : Enums.toEnum(record.get(ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, String.class), SupportLevel.class).orElseThrow()) .withProtocolVersion(AirbyteProtocolVersion.getWithDefault(record.get(ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION)).serialize()) .withReleaseStage(record.get(ACTOR_DEFINITION_VERSION.RELEASE_STAGE) == null ? null : Enums.toEnum(record.get(ACTOR_DEFINITION_VERSION.RELEASE_STAGE, String.class), ReleaseStage.class).orElseThrow()) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java index 21025fa85d1..aa59ced8f1e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java @@ -15,6 +15,7 @@ import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; @@ -25,13 +26,16 @@ * Handle persisting Permission to the Config Database and perform all SQL queries. * */ +@Slf4j public class OrganizationPersistence { private final ExceptionWrappingDatabase database; - public static final String PRIMARY_KEY = "id"; - public static final String USER_KEY = "user_id"; - public static final String WORKSPACE_KEY = "workspace_id"; + /** + * Each installation of Airbyte comes with a default organization. The ID of this organization is + * hardcoded to the 0 UUID so that it can be consistently retrieved. + */ + public static final UUID DEFAULT_ORGANIZATION_ID = UUID.fromString("00000000-0000-0000-0000-000000000000"); public OrganizationPersistence(final Database database) { this.database = new ExceptionWrappingDatabase(database); @@ -95,6 +99,13 @@ public Organization updateOrganization(Organization organization) throws IOExcep return organization; } + /** + * Get the default organization if it exists by looking up the hardcoded default organization id. + */ + public Optional getDefaultOrganization() throws IOException { + return getOrganization(DEFAULT_ORGANIZATION_ID); + } + private void updateOrganizationInDB(final DSLContext ctx, Organization organization) throws IOException { final OffsetDateTime timestamp = OffsetDateTime.now(); @@ -124,6 +135,7 @@ private void insertOrganizationIntoDB(final DSLContext ctx, Organization organiz } ctx.insertInto(ORGANIZATION) .set(ORGANIZATION.ID, organization.getOrganizationId()) + .set(ORGANIZATION.USER_ID, organization.getUserId()) .set(ORGANIZATION.NAME, organization.getName()) .set(ORGANIZATION.EMAIL, organization.getEmail()) .set(ORGANIZATION.CREATED_AT, timestamp) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java index d8ef9a455ea..9aa27cb2ffa 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java @@ -14,6 +14,7 @@ import io.airbyte.config.Permission; import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.UserPermission; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; @@ -196,9 +197,101 @@ public List listUsersInWorkspace(final UUID workspaceId) throws return this.database.query(ctx -> listPermissionsForWorkspace(ctx, workspaceId)); } + public List listInstanceAdminUsers() throws IOException { + return this.database.query(ctx -> listInstanceAdminPermissions(ctx)); + } + public List listUsersInOrganization(final UUID organizationId) throws IOException { return this.database.query(ctx -> listPermissionsForOrganization(ctx, organizationId)); + } + + private List listInstanceAdminPermissions(final DSLContext ctx) { + var records = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) + .fetch(); + + return records.stream().map(record -> buildUserPermissionFromRecord(record)).collect(Collectors.toList()); + } + + private UserPermission getUserInstanceAdminPermission(final DSLContext ctx, final UUID userId) { + var record = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) + .and(PERMISSION.USER_ID.eq(userId)) + .fetchOne(); + if (record == null) { + return null; + } + return buildUserPermissionFromRecord(record); + } + + /** + * Check and get instance_admin permission for a user. + * + * @param userId user id + * @return UserPermission User details with instance_admin permission, null if user does not have + * instance_admin role. + * @throws IOException if there is an issue while interacting with the db. + */ + public UserPermission getUserInstanceAdminPermission(final UUID userId) throws IOException { + return this.database.query(ctx -> getUserInstanceAdminPermission(ctx, userId)); + } + + public PermissionType findPermissionTypeForUserAndWorkspace(final UUID workspaceId, final String authUserId, final AuthProvider authProvider) + throws IOException { + return this.database.query(ctx -> findPermissionTypeForUserAndWorkspace(ctx, workspaceId, authUserId, authProvider)); + } + + private PermissionType findPermissionTypeForUserAndWorkspace(final DSLContext ctx, + final UUID workspaceId, + final String authUserId, + final AuthProvider authProvider) { + var record = ctx.select(PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.WORKSPACE_ID.eq(workspaceId)) + .and(USER.AUTH_USER_ID.eq(authUserId)) + .and(USER.AUTH_PROVIDER.eq(Enums.toEnum(authProvider.value(), io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider.class).get())) + .fetchOne(); + if (record == null) { + return null; + } + + final var jooqPermissionType = record.get(PERMISSION.PERMISSION_TYPE, io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.class); + + return Enums.toEnum(jooqPermissionType.getLiteral(), PermissionType.class).get(); + } + + public PermissionType findPermissionTypeForUserAndOrganization(final UUID organizationId, final String authUserId, final AuthProvider authProvider) + throws IOException { + return this.database.query(ctx -> findPermissionTypeForUserAndOrganization(ctx, organizationId, authUserId, authProvider)); + } + + private PermissionType findPermissionTypeForUserAndOrganization(final DSLContext ctx, + final UUID organizationId, + final String authUserId, + final AuthProvider authProvider) { + var record = ctx.select(PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.ORGANIZATION_ID.eq(organizationId)) + .and(USER.AUTH_USER_ID.eq(authUserId)) + .and(USER.AUTH_PROVIDER.eq(Enums.toEnum(authProvider.value(), io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider.class).get())) + .fetchOne(); + + if (record == null) { + return null; + } + final var jooqPermissionType = record.get(PERMISSION.PERMISSION_TYPE, io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.class); + return Enums.toEnum(jooqPermissionType.getLiteral(), PermissionType.class).get(); } private List listPermissionsForWorkspace(final DSLContext ctx, final UUID workspaceId) { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java index 317267335ca..ab973503849 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java @@ -4,22 +4,39 @@ package io.airbyte.config.persistence; +import static io.airbyte.featureflag.ContextKt.ANONYMOUS; + import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; +import io.airbyte.config.Configs.DeploymentMode; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.PauseSyncsWithUnsupportedActors; +import io.airbyte.featureflag.Workspace; +import io.airbyte.validation.json.JsonValidationException; import jakarta.inject.Singleton; import java.io.IOException; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -49,9 +66,28 @@ public static SupportStateUpdate merge(final SupportStateUpdate a, final Support } private final ConfigRepository configRepository; - - public SupportStateUpdater(final ConfigRepository configRepository) { + private final ActorDefinitionVersionHelper actorDefinitionVersionHelper; + private final DeploymentMode deploymentMode; + private final FeatureFlagClient featureFlagClient; + + public SupportStateUpdater(final ConfigRepository configRepository, + final ActorDefinitionVersionHelper actorDefinitionVersionHelper, + final DeploymentMode deploymentMode, + final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; + this.actorDefinitionVersionHelper = actorDefinitionVersionHelper; + this.deploymentMode = deploymentMode; + this.featureFlagClient = featureFlagClient; + } + + @VisibleForTesting + boolean shouldDisableSyncs() { + if (!featureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) { + return false; + } + + // We only disable syncs on Cloud. OSS users can continue to run on unsupported versions. + return deploymentMode == DeploymentMode.CLOUD; } /** @@ -133,11 +169,15 @@ private SupportState calcVersionSupportState(final Version version, /** * Updates the version support states for a given source definition. */ - public void updateSupportStatesForSourceDefinition(final StandardSourceDefinition sourceDefinition) throws ConfigNotFoundException, IOException { + public void updateSupportStatesForSourceDefinition(final StandardSourceDefinition sourceDefinition) + throws ConfigNotFoundException, IOException { if (!sourceDefinition.getCustom()) { + log.info("Updating support states for source definition: {}", sourceDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(sourceDefinition.getSourceDefinitionId(), currentDefaultVersion); + + log.info("Finished updating support states for source definition: {}", sourceDefinition.getName()); } } @@ -147,10 +187,13 @@ public void updateSupportStatesForSourceDefinition(final StandardSourceDefinitio public void updateSupportStatesForDestinationDefinition(final StandardDestinationDefinition destinationDefinition) throws ConfigNotFoundException, IOException { if (!destinationDefinition.getCustom()) { + log.info("Updating support states for destination definition: {}", destinationDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(destinationDefinition.getDestinationDefinitionId(), currentDefaultVersion); + + log.info("Finished updating support states for destination definition: {}", destinationDefinition.getName()); } } @@ -170,39 +213,176 @@ private Version getVersionTag(final List actorDefinition .orElseThrow(); } + /** + * Calculates unsupported version IDs after the support state update would be applied. + *

+ * This is done by taking the set of all previously unsupported versions, adding newly unsupported + * versions, and removing any versions that would be supported or deprecated after the update. + * + * @param versionsBeforeUpdate - actor definition versions before applying the SupportStateUpdate + * @param supportStateUpdate - the SupportStateUpdate that would be applied + * @return - unsupported actor definition version IDs + */ + @VisibleForTesting + List getUnsupportedVersionIdsAfterUpdate(final List versionsBeforeUpdate, + final SupportStateUpdate supportStateUpdate) { + final List prevUnsupportedVersionIds = versionsBeforeUpdate.stream() + .filter(v -> v.getSupportState() == SupportState.UNSUPPORTED) + .map(ActorDefinitionVersion::getVersionId) + .filter(verId -> !supportStateUpdate.supportedVersionIds().contains(verId) && !supportStateUpdate.deprecatedVersionIds().contains(verId)) + .toList(); + return Stream.of(prevUnsupportedVersionIds, supportStateUpdate.unsupportedVersionIds()) + .flatMap(Collection::stream) + .toList(); + } + /** * Updates the version support states for all source and destination definitions. */ - public void updateSupportStates() throws IOException { + public void updateSupportStates() throws IOException, JsonValidationException, ConfigNotFoundException { + updateSupportStates(LocalDate.now()); + } + + /** + * Updates the version support states for all source and destination definitions based on a + * reference date, and disables syncs with unsupported versions. + */ + @VisibleForTesting + void updateSupportStates(final LocalDate referenceDate) throws IOException, JsonValidationException, ConfigNotFoundException { + log.info("Updating support states for all definitions"); final List sourceDefinitions = configRepository.listPublicSourceDefinitions(false); final List destinationDefinitions = configRepository.listPublicDestinationDefinitions(false); - final List breakingChanges = configRepository.listBreakingChanges(); + final List allBreakingChanges = configRepository.listBreakingChanges(); + final Map> breakingChangesMap = allBreakingChanges.stream() + .collect(Collectors.groupingBy(ActorDefinitionBreakingChange::getActorDefinitionId)); SupportStateUpdate comboSupportStateUpdate = new SupportStateUpdate(List.of(), List.of(), List.of()); + final List syncsToDisable = new ArrayList<>(); for (final StandardSourceDefinition sourceDefinition : sourceDefinitions) { final List actorDefinitionVersions = configRepository.listActorDefinitionVersionsForDefinition(sourceDefinition.getSourceDefinitionId()); final Version currentDefaultVersion = getVersionTag(actorDefinitionVersions, sourceDefinition.getDefaultVersionId()); + final List breakingChangesForDef = + breakingChangesMap.getOrDefault(sourceDefinition.getSourceDefinitionId(), List.of()); final SupportStateUpdate supportStateUpdate = - getSupportStateUpdate(currentDefaultVersion, LocalDate.now(), breakingChanges, actorDefinitionVersions); - + getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); + + final List unsupportedVersionIds = getUnsupportedVersionIdsAfterUpdate(actorDefinitionVersions, supportStateUpdate); + final List syncsToDisableForSource = getSyncsToDisableForSource(sourceDefinition, unsupportedVersionIds); + syncsToDisable.addAll(syncsToDisableForSource); } for (final StandardDestinationDefinition destinationDefinition : destinationDefinitions) { final List actorDefinitionVersions = configRepository.listActorDefinitionVersionsForDefinition(destinationDefinition.getDestinationDefinitionId()); final Version currentDefaultVersion = getVersionTag(actorDefinitionVersions, destinationDefinition.getDefaultVersionId()); + final List breakingChangesForDef = + breakingChangesMap.getOrDefault(destinationDefinition.getDestinationDefinitionId(), List.of()); final SupportStateUpdate supportStateUpdate = - getSupportStateUpdate(currentDefaultVersion, LocalDate.now(), breakingChanges, actorDefinitionVersions); - + getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); + + final List unsupportedVersionIds = getUnsupportedVersionIdsAfterUpdate(actorDefinitionVersions, supportStateUpdate); + final List syncsToDisableForDestination = getSyncsToDisableForDestination(destinationDefinition, unsupportedVersionIds); + syncsToDisable.addAll(syncsToDisableForDestination); } executeSupportStateUpdate(comboSupportStateUpdate); + disableSyncs(syncsToDisable); + log.info("Finished updating support states for all definitions"); + } + + @VisibleForTesting + List getSyncsToDisableForSource(final StandardSourceDefinition sourceDefinition, final List unsupportedVersionIds) + throws IOException, JsonValidationException, ConfigNotFoundException { + if (!shouldDisableSyncs() || unsupportedVersionIds.isEmpty()) { + return Collections.emptyList(); + } + + final List syncsToDisable = new ArrayList<>(); + final List sourceConnections = configRepository.listSourcesWithVersionIds(unsupportedVersionIds); + final Map> sourceConnectionsByWorkspace = new HashMap<>(); + + // verify that a version override has not been applied to the source, and collect by workspace + for (final SourceConnection source : sourceConnections) { + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, source.getWorkspaceId(), source.getSourceId()); + if (!versionWithOverrideStatus.isOverrideApplied()) { + sourceConnectionsByWorkspace + .computeIfAbsent(source.getWorkspaceId(), k -> new ArrayList<>()) + .add(source); + } + } + + // get affected syncs for each workspace and add them to the list + for (final Map.Entry> entry : sourceConnectionsByWorkspace.entrySet()) { + final UUID workspaceId = entry.getKey(); + final List sourcesForWorkspace = entry.getValue(); + final List sourceIds = sourcesForWorkspace.stream().map(SourceConnection::getSourceId).toList(); + final StandardSyncQuery syncQuery = new StandardSyncQuery(workspaceId, sourceIds, null, false); + final List standardSyncs = configRepository.listWorkspaceStandardSyncs(syncQuery); + syncsToDisable.addAll(standardSyncs); + } + + return syncsToDisable; + } + + @VisibleForTesting + List getSyncsToDisableForDestination(final StandardDestinationDefinition destinationDefinition, + final List unsupportedVersionIds) + throws IOException, JsonValidationException, ConfigNotFoundException { + if (!shouldDisableSyncs() || unsupportedVersionIds.isEmpty()) { + return Collections.emptyList(); + } + + final List syncsToDisable = new ArrayList<>(); + final List destinationConnections = configRepository.listDestinationsWithVersionIds(unsupportedVersionIds); + final Map> destinationConnectionsByWorkspace = new HashMap<>(); + + // verify that a version override has not been applied to the destination, and collect by workspace + for (final DestinationConnection destination : destinationConnections) { + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus( + destinationDefinition, + destination.getWorkspaceId(), + destination.getDestinationId()); + if (!versionWithOverrideStatus.isOverrideApplied()) { + destinationConnectionsByWorkspace + .computeIfAbsent(destination.getWorkspaceId(), k -> new ArrayList<>()) + .add(destination); + } + } + + // get affected syncs for each workspace and add them to the list + for (final Map.Entry> entry : destinationConnectionsByWorkspace.entrySet()) { + final UUID workspaceId = entry.getKey(); + final List destinationsForWorkspace = entry.getValue(); + final List destinationIds = destinationsForWorkspace.stream().map(DestinationConnection::getDestinationId).toList(); + final StandardSyncQuery syncQuery = new StandardSyncQuery(workspaceId, null, destinationIds, false); + final List standardSyncs = configRepository.listWorkspaceStandardSyncs(syncQuery); + syncsToDisable.addAll(standardSyncs); + } + + return syncsToDisable; + } + + @VisibleForTesting + void disableSyncs(final List syncsToDisable) throws IOException { + final List activeSyncs = syncsToDisable.stream() + .filter(s -> s.getStatus() == Status.ACTIVE) + .toList(); + + for (final StandardSync sync : activeSyncs) { + configRepository.writeStandardSync(sync.withStatus(Status.INACTIVE)); + } + + if (!activeSyncs.isEmpty()) { + log.info("Disabled {} syncs with unsupported versions", activeSyncs.size()); + } } /** @@ -211,8 +391,6 @@ public void updateSupportStates() throws IOException { * @param supportStateUpdate - the SupportStateUpdate to process. */ private void executeSupportStateUpdate(final SupportStateUpdate supportStateUpdate) throws IOException { - // TODO(pedro): This is likely where we disable syncs for the now-unsupported versions. - if (!supportStateUpdate.unsupportedVersionIds.isEmpty()) { configRepository.setActorDefinitionVersionSupportStates(supportStateUpdate.unsupportedVersionIds, ActorDefinitionVersion.SupportState.UNSUPPORTED); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java index 46e42700b30..6448cbf3816 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java @@ -19,6 +19,7 @@ import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.jooq.Record; import org.jooq.Result; import org.jooq.impl.DSL; @@ -29,10 +30,17 @@ * Perform all SQL queries and handle persisting User to the Config Database. * */ +@Slf4j public class UserPersistence { public static final String PRIMARY_KEY = "id"; + /** + * Each installation of Airbyte comes with a default user. The ID of this user is hardcoded to the 0 + * UUID so that it can be consistently retrieved. + */ + public static final UUID DEFAULT_USER_ID = UUID.fromString("00000000-0000-0000-0000-000000000000"); + private final ExceptionWrappingDatabase database; public UserPersistence(final Database database) { @@ -177,4 +185,11 @@ public Optional getUserByEmail(final String email) throws IOException { return Optional.of(createUserFromRecord(result.get(0))); } + /** + * Get the default user if it exists by looking up the hardcoded default user id. + */ + public Optional getDefaultUser() throws IOException { + return getUser(DEFAULT_USER_ID); + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java index 5a1beae56a2..ba945f12c1e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java @@ -9,15 +9,19 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; /** * Persistence Interface for Workspace table. */ +@Slf4j public class WorkspacePersistence { private final ExceptionWrappingDatabase database; @@ -26,16 +30,56 @@ public WorkspacePersistence(final Database database) { this.database = new ExceptionWrappingDatabase(database); } + /** + * List all workspaces as user has instance_admin role. Returning result ordered by workspace name. + * Supports pagination and keyword search. + */ + public List listWorkspacesByInstanceAdminUserPaginated(final boolean includeDeleted, + final int pageSize, + final int rowOffset, + Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .limit(pageSize) + .offset(rowOffset) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * List all workspaces as user has instance_admin role. Returning result ordered by workspace name. + * Supports keyword search. + */ + public List listWorkspacesByInstanceAdminUser(final boolean includeDeleted, Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + /** * List all workspaces owned by org id, returning result ordered by workspace name. Supports * pagination and keyword search. */ - public List listWorkspacesByOrganizationId(final ResourcesByOrganizationQueryPaginated query, Optional keyword) + public List listWorkspacesByOrganizationIdPaginated(final ResourcesByOrganizationQueryPaginated query, Optional keyword) throws IOException { return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) .from(WORKSPACE) .where(WORKSPACE.ORGANIZATION_ID.eq(query.organizationId())) .and(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(query.includeDeleted() ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) .orderBy(WORKSPACE.NAME.asc()) .limit(query.pageSize()) .offset(query.rowOffset()) @@ -45,4 +89,97 @@ public List listWorkspacesByOrganizationId(final ResourcesByO .toList(); } + /** + * List all workspaces owned by org id, returning result ordered by workspace name. Supports keyword + * search. + */ + public List listWorkspacesByOrganizationId(UUID organizationId, boolean includeDeleted, Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.ORGANIZATION_ID.eq(organizationId)) + .and(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * This query is to list all workspaces that a user has read permissions. + */ + private final String listWorkspacesByUserIdBasicQuery = + "WITH userOrgs AS (SELECT organization_id FROM permission WHERE user_id = {0})," + + " userWorkspaces AS (" + + " SELECT workspace.id AS workspace_id FROM userOrgs JOIN workspace" + + " ON workspace.organization_id = userOrgs.organization_id" + + " UNION" + + " SELECT workspace_id FROM permission WHERE user_id = {1}" + + " )" + + " SELECT * from workspace" + + " WHERE workspace.id IN (SELECT workspace_id from userWorkspaces)" + + " AND name ILIKE {2}" + + " AND tombstone = false" + + " ORDER BY name ASC"; + + /** + * Get search keyword with flexible matching. + */ + private String getSearchKeyword(Optional keyword) { + if (keyword.isPresent()) { + return "%" + keyword.get().toLowerCase() + "%"; + } else { + return "%%"; + } + } + + /** + * List all workspaces owned by user id, returning result ordered by workspace name. Supports + * keyword search. + */ + public List listWorkspacesByUserId(UUID userId, boolean includeDeleted, Optional keyword) + throws IOException { + final String searchKeyword = getSearchKeyword(keyword); + return database.query(ctx -> ctx.fetch(listWorkspacesByUserIdBasicQuery, userId, userId, searchKeyword)) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * List all workspaces owned by user id, returning result ordered by workspace name. Supports + * pagination and keyword search. + */ + public List listWorkspacesByUserIdPaginated(final ResourcesByUserQueryPaginated query, Optional keyword) + throws IOException { + final String searchKeyword = getSearchKeyword(keyword); + final String workspaceQuery = listWorkspacesByUserIdBasicQuery + + " LIMIT {3}" + + " OFFSET {4}"; + + return database.query(ctx -> ctx.fetch(workspaceQuery, query.userId(), query.userId(), searchKeyword, query.pageSize(), query.rowOffset())) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * Fetch the oldest, non-tombstoned Workspace that belongs to the given Organization. + */ + public StandardWorkspace getDefaultWorkspaceForOrganization(final UUID organizationId) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.ORGANIZATION_ID.eq(organizationId)) + .and(WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.CREATED_AT.asc()) + .limit(1) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .findFirst() + .orElseThrow(() -> new RuntimeException("No workspace found for organization: " + organizationId)); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java index 676d0e85d39..d724f15351f 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -13,6 +14,7 @@ import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -71,6 +73,16 @@ final ActorDefinitionVersion createActorDefVersion(final UUID actorDefinitionId) .withActorDefinitionId(actorDefinitionId) .withDockerImageTag("1.0.0") .withDockerRepository("repo") + .withSupportLevel(SupportLevel.COMMUNITY) + .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); + } + + final ActorDefinitionVersion createActorDefVersion(final UUID actorDefinitionId, final String dockerImageTag) { + return new ActorDefinitionVersion() + .withActorDefinitionId(actorDefinitionId) + .withDockerImageTag(dockerImageTag) + .withDockerRepository("repo") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); } @@ -82,32 +94,25 @@ void setup() throws SQLException, JsonValidationException, IOException { configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId())); - configRepository.writeDestinationDefinitionAndDefaultVersion(DESTINATION_DEFINITION, - createActorDefVersion(DESTINATION_DEFINITION.getDestinationDefinitionId())); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId()), + List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); + configRepository.writeConnectorMetadata(DESTINATION_DEFINITION, + createActorDefVersion(DESTINATION_DEFINITION.getDestinationDefinitionId()), List.of(OTHER_CONNECTOR_BREAKING_CHANGE)); } @Test - void testReadAndWriteActorDefinitionBreakingChange() throws IOException { - final List prevBreakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(0, prevBreakingChanges.size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE)); - final List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); - assertEquals(BREAKING_CHANGE, breakingChanges.get(0)); + void testGetBreakingChanges() throws IOException { + final List breakingChangesForDef1 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); + assertEquals(4, breakingChangesForDef1.size()); + assertEquals(BREAKING_CHANGE, breakingChangesForDef1.get(0)); + + final List breakingChangesForDef2 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2); + assertEquals(1, breakingChangesForDef2.size()); + assertEquals(OTHER_CONNECTOR_BREAKING_CHANGE, breakingChangesForDef2.get(0)); } @Test void testUpdateActorDefinitionBreakingChange() throws IOException { - final List prevBreakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(0, prevBreakingChanges.size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE)); - List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); - assertEquals(BREAKING_CHANGE, breakingChanges.get(0)); - // Update breaking change final ActorDefinitionBreakingChange updatedBreakingChange = new ActorDefinitionBreakingChange() .withActorDefinitionId(BREAKING_CHANGE.getActorDefinitionId()) @@ -115,69 +120,42 @@ void testUpdateActorDefinitionBreakingChange() throws IOException { .withMessage("Updated message") .withUpgradeDeadline("2025-01-01") .withMigrationDocumentationUrl(BREAKING_CHANGE.getMigrationDocumentationUrl()); - configRepository.writeActorDefinitionBreakingChanges(List.of(updatedBreakingChange)); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId()), + List.of(updatedBreakingChange, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); // Check updated breaking change - breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); + final List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); + assertEquals(4, breakingChanges.size()); assertEquals(updatedBreakingChange, breakingChanges.get(0)); } - @Test - void testWriteMultipleActorDefinitionBreakingChanges() throws IOException { - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2).size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, OTHER_CONNECTOR_BREAKING_CHANGE)); - - final List breakingChangesForId1 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(2, breakingChangesForId1.size()); - assertEquals(BREAKING_CHANGE, breakingChangesForId1.get(0)); - assertEquals(BREAKING_CHANGE_2, breakingChangesForId1.get(1)); - - final List breakingChangesForId2 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2); - assertEquals(1, breakingChangesForId2.size()); - assertEquals(OTHER_CONNECTOR_BREAKING_CHANGE, breakingChangesForId2.get(0)); - } - @Test void testListBreakingChanges() throws IOException { - final List breakingChanges = List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4); - assertEquals(0, configRepository.listBreakingChanges().size()); - configRepository.writeActorDefinitionBreakingChanges(breakingChanges); - assertEquals(breakingChanges, configRepository.listBreakingChanges()); + final List expectedAllBreakingChanges = + List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4, OTHER_CONNECTOR_BREAKING_CHANGE); + assertThat(expectedAllBreakingChanges).containsExactlyInAnyOrderElementsOf(configRepository.listBreakingChanges()); } @Test void testListBreakingChangesForVersion() throws IOException { - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); - - final ActorDefinitionVersion ADV_4_0_0 = createActorDefinitionVersion("4.0.0"); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ADV_4_0_0); + final ActorDefinitionVersion ADV_4_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "4.0.0"); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, ADV_4_0_0); // no breaking changes for latest default assertEquals(4, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); assertEquals(0, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_4_0_0).size()); // should see future breaking changes for 2.0.0 - final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion("2.0.0"); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "2.0.0"); assertEquals(2, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0).size()); assertEquals(List.of(BREAKING_CHANGE_3, BREAKING_CHANGE_4), configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0)); // move back default version for Actor Definition to 3.0.0, should stop seeing "rolled back" // breaking changes - final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion("3.0.0"); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ADV_3_0_0); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "3.0.0"); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, ADV_3_0_0); assertEquals(1, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0).size()); assertEquals(List.of(BREAKING_CHANGE_3), configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0)); } - ActorDefinitionVersion createActorDefinitionVersion(final String version) { - return new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID_1) - .withDockerRepository("some-repo") - .withDockerImageTag(version); - } - } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java index c155cefa73b..554e25b9649 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java @@ -4,18 +4,14 @@ package io.airbyte.config.persistence; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import io.airbyte.commons.version.Version; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; @@ -24,18 +20,17 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; class ActorDefinitionPersistenceTest extends BaseConfigDatabaseTest { @@ -43,8 +38,6 @@ class ActorDefinitionPersistenceTest extends BaseConfigDatabaseTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final String DOCKER_IMAGE_TAG = "0.0.1"; - private static final String UPDATED_IMAGE_TAG = "0.0.2"; - private ConfigRepository configRepository; @BeforeEach @@ -81,7 +74,7 @@ void testSourceDefinitionMaxSeconds() throws JsonValidationException, ConfigNotF private void assertReturnsSrcDef(final StandardSourceDefinition srcDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); assertEquals(srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), configRepository.getStandardSourceDefinition(srcDef.getSourceDefinitionId())); } @@ -89,7 +82,7 @@ private void assertReturnsSrcDef(final StandardSourceDefinition srcDef) throws C private void assertReturnsSrcDefDefaultMaxSecondsBetweenMessages(final StandardSourceDefinition srcDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); assertEquals( srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()) .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES), @@ -97,13 +90,13 @@ private void assertReturnsSrcDefDefaultMaxSecondsBetweenMessages(final StandardS } @Test - void testSourceDefinitionFromSource() throws JsonValidationException, IOException { + void testGetSourceDefinitionFromSource() throws JsonValidationException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardSourceDefinition srcDef = createBaseSourceDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); final SourceConnection source = createSource(srcDef.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(source); assertEquals(srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), @@ -111,13 +104,13 @@ void testSourceDefinitionFromSource() throws JsonValidationException, IOExceptio } @Test - void testSourceDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetSourceDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardSourceDefinition srcDef = createBaseSourceDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); final SourceConnection source = createSource(srcDef.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(source); final UUID connectionId = UUID.randomUUID(); @@ -147,7 +140,7 @@ void testListStandardSourceDefsHandlesTombstoneSourceDefs(final int numSrcDefs) if (!isTombstone) { notTombstoneSourceDefinitions.add(sourceDefinition); } - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); sourceDefinition.setDefaultVersionId(actorDefinitionVersion.getVersionId()); } @@ -175,19 +168,19 @@ void testDestinationDefinitionWithFalseTombstone() throws JsonValidationExceptio void assertReturnsDestDef(final StandardDestinationDefinition destDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); assertEquals(destDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), configRepository.getStandardDestinationDefinition(destDef.getDestinationDefinitionId())); } @Test - void testDestinationDefinitionFromDestination() throws JsonValidationException, IOException { + void testGetDestinationDefinitionFromDestination() throws JsonValidationException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardDestinationDefinition destDef = createBaseDestDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); final DestinationConnection dest = createDest(destDef.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); configRepository.writeDestinationConnectionNoSecrets(dest); assertEquals(destDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), @@ -195,13 +188,13 @@ void testDestinationDefinitionFromDestination() throws JsonValidationException, } @Test - void testDestinationDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetDestinationDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardDestinationDefinition destDef = createBaseDestDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); final DestinationConnection dest = createDest(destDef.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); configRepository.writeDestinationConnectionNoSecrets(dest); final UUID connectionId = UUID.randomUUID(); @@ -231,7 +224,7 @@ void testListStandardDestDefsHandlesTombstoneDestDefs(final int numDestinationDe if (!isTombstone) { notTombstoneDestinationDefinitions.add(destinationDefinition); } - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); destinationDefinition.setDefaultVersionId(actorDefinitionVersion.getVersionId()); } @@ -255,9 +248,9 @@ void testUpdateAllImageTagsForDeclarativeSourceDefinition() throws JsonValidatio sourceVer2.setDockerImageTag(targetImageTag); final ActorDefinitionVersion sourceVer3 = createBaseActorDefVersion(sourceDef3.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef1, sourceVer1); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef2, sourceVer2); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef3, sourceVer3); + configRepository.writeConnectorMetadata(sourceDef1, sourceVer1); + configRepository.writeConnectorMetadata(sourceDef2, sourceVer2); + configRepository.writeConnectorMetadata(sourceDef3, sourceVer3); final int updatedDefinitions = configRepository .updateActorDefinitionsDockerImageTag(List.of(sourceDef1.getSourceDefinitionId(), sourceDef2.getSourceDefinitionId()), targetImageTag); @@ -281,23 +274,23 @@ void getActorDefinitionIdsInUse() throws IOException, JsonValidationException { final StandardSourceDefinition sourceDefInUse = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion3 = createBaseActorDefVersion(sourceDefInUse.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefInUse, actorDefinitionVersion3); + configRepository.writeConnectorMetadata(sourceDefInUse, actorDefinitionVersion3); final SourceConnection sourceConnection = createSource(sourceDefInUse.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeSourceConnectionNoSecrets(sourceConnection); final StandardSourceDefinition sourceDefNotInUse = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion4 = createBaseActorDefVersion(sourceDefNotInUse.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefNotInUse, actorDefinitionVersion4); + configRepository.writeConnectorMetadata(sourceDefNotInUse, actorDefinitionVersion4); final StandardDestinationDefinition destDefInUse = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDefInUse.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDefInUse, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDefInUse, actorDefinitionVersion); final DestinationConnection destinationConnection = createDest(destDefInUse.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeDestinationConnectionNoSecrets(destinationConnection); final StandardDestinationDefinition destDefNotInUse = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(destDefNotInUse.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDefNotInUse, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destDefNotInUse, actorDefinitionVersion2); assertTrue(configRepository.getActorDefinitionIdsInUse().contains(sourceDefInUse.getSourceDefinitionId())); assertTrue(configRepository.getActorDefinitionIdsInUse().contains(destDefInUse.getDestinationDefinitionId())); @@ -309,11 +302,11 @@ void getActorDefinitionIdsInUse() throws IOException, JsonValidationException { void testGetActorDefinitionIdsToDefaultVersionsMap() throws IOException { final StandardSourceDefinition sourceDef = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDef, actorDefinitionVersion); final StandardDestinationDefinition destDef = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion2); final Map actorDefIdToDefaultVersionId = configRepository.getActorDefinitionIdsToDefaultVersionsMap(); assertEquals(actorDefIdToDefaultVersionId.size(), 2); @@ -321,158 +314,12 @@ void testGetActorDefinitionIdsToDefaultVersionsMap() throws IOException { assertEquals(actorDefIdToDefaultVersionId.get(destDef.getDestinationDefinitionId()), actorDefinitionVersion2); } - @Test - void testWriteSourceDefinitionAndDefaultVersion() - throws JsonValidationException, IOException, ConfigNotFoundException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion1); - - StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); - final Optional actorDefinitionVersionFromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); - - assertTrue(actorDefinitionVersionFromDB.isPresent()); - final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); - - assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); - assertEquals(firstVersionId, sourceDefinitionFromDB.getDefaultVersionId()); - assertEquals(sourceDefinition.withDefaultVersionId(firstVersionId), sourceDefinitionFromDB); - - // Updating an existing source definition/version - final StandardSourceDefinition sourceDefinition2 = sourceDefinition.withName("updated name"); - final ActorDefinitionVersion actorDefinitionVersion2 = - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(UPDATED_IMAGE_TAG); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition2, actorDefinitionVersion2); - - sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); - final Optional actorDefinitionVersion2FromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); - - assertTrue(actorDefinitionVersion2FromDB.isPresent()); - final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); - - assertNotEquals(firstVersionId, newADVId); - assertEquals(newADVId, sourceDefinitionFromDB.getDefaultVersionId()); - assertEquals(sourceDefinition2.withDefaultVersionId(newADVId), sourceDefinitionFromDB); - } - - @Test - void testWriteDestinationDefinitionAndDefaultVersion() - throws JsonValidationException, IOException, ConfigNotFoundException { - // Initial insert - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion1); - - StandardDestinationDefinition destinationDefinitionFromDB = - configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); - final Optional actorDefinitionVersionFromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); - - assertTrue(actorDefinitionVersionFromDB.isPresent()); - final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); - - assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); - assertEquals(firstVersionId, destinationDefinitionFromDB.getDefaultVersionId()); - assertEquals(destinationDefinition.withDefaultVersionId(firstVersionId), destinationDefinitionFromDB); - - // Updating an existing destination definition/version - final StandardDestinationDefinition destinationDefinition2 = destinationDefinition.withName("updated name"); - final ActorDefinitionVersion actorDefinitionVersion2 = - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(UPDATED_IMAGE_TAG); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition2, actorDefinitionVersion2); - - destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); - final Optional actorDefinitionVersion2FromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); - - assertTrue(actorDefinitionVersion2FromDB.isPresent()); - final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); - - assertNotEquals(firstVersionId, newADVId); - assertEquals(newADVId, destinationDefinitionFromDB.getDefaultVersionId()); - assertEquals(destinationDefinition2.withDefaultVersionId(newADVId), destinationDefinitionFromDB); - } - - @ParameterizedTest - @ValueSource(strings = {"1.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) - void testCustomImageTagsDoNotBreakCustomConnectorUpgrade(final String dockerImageTag) throws IOException { - // Initial insert - final StandardSourceDefinition customSourceDefinition = createBaseSourceDef().withCustom(true); - final StandardDestinationDefinition customDestinationDefinition = createBaseDestDef().withCustom(true); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = - createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(customSourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(customDestinationDefinition, destinationActorDefinitionVersion); - - // Update - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(customSourceDefinition, - createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(customDestinationDefinition, - createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - } - - @ParameterizedTest - @ValueSource(strings = {"1.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) - void testImageTagExpectationsNorNonCustomConnectorUpgradesWithoutBreakingChanges(final String dockerImageTag) throws IOException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, destinationActorDefinitionVersion); - - // Update - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - } - - @ParameterizedTest - @CsvSource({"0.0.1, true", "dev, true", "test, false", "1.9.1-dev.33a53e6236, true", "97b69a76-1f06-4680-8905-8beda74311d0, false"}) - void testImageTagExpectationsNorNonCustomConnectorUpgradesWithBreakingChanges(final String dockerImageTag, final boolean upgradeShouldSucceed) - throws IOException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, destinationActorDefinitionVersion); - - final List sourceBreakingChanges = createBreakingChangesForDef(sourceDefinition.getSourceDefinitionId()); - final List destinationBreakingChanges = - createBreakingChangesForDef(destinationDefinition.getDestinationDefinitionId()); - - // Update - if (upgradeShouldSucceed) { - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), - destinationBreakingChanges)); - } else { - assertThrows(IllegalArgumentException.class, () -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); - assertThrows(IllegalArgumentException.class, () -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), - destinationBreakingChanges)); - } - } - @Test void testUpdateStandardSourceDefinition() throws IOException, JsonValidationException, ConfigNotFoundException { final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); @@ -505,7 +352,7 @@ void testUpdateStandardDestinationDefinition() throws IOException, JsonValidatio final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); final StandardDestinationDefinition destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); @@ -567,18 +414,10 @@ private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actor .withActorDefinitionId(actorDefId) .withDockerRepository("source-image-" + actorDefId) .withDockerImageTag(DOCKER_IMAGE_TAG) + .withSupportLevel(SupportLevel.COMMUNITY) .withProtocolVersion("0.2.0"); } - private List createBreakingChangesForDef(final UUID actorDefId) { - return List.of(new ActorDefinitionBreakingChange() - .withActorDefinitionId(actorDefId) - .withVersion(new Version("1.0.0")) - .withMessage("This is a breaking change") - .withMigrationDocumentationUrl("https://docs.airbyte.com/migration#1.0.0") - .withUpgradeDeadline("2025-01-21")); - } - private static StandardSourceDefinition createBaseSourceDefWithoutMaxSecondsBetweenMessages() { final UUID id = UUID.randomUUID(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java index fe27b66c362..f31e1138c6d 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java @@ -5,6 +5,7 @@ package io.airbyte.config.persistence; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -20,6 +21,7 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.version_overrides.DefinitionVersionOverrideProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.TestClient; @@ -94,8 +96,10 @@ void testGetSourceVersion() throws ConfigNotFoundException, IOException, JsonVal .withSourceDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -107,8 +111,10 @@ void testGetSourceVersionFromActorDefault() throws ConfigNotFoundException, IOEx when(mFeatureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(WORKSPACE_ID))).thenReturn(true); when(mConfigRepository.getSourceConnection(ACTOR_ID)).thenReturn(new SourceConnection().withDefaultVersionId(DEFAULT_VERSION_ID)); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -120,8 +126,10 @@ void testGetSourceVersionWithOverride() throws ConfigNotFoundException, IOExcept .withSourceDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(OVERRIDDEN_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(OVERRIDDEN_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertTrue(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -165,8 +173,10 @@ void testGetDestinationVersion() throws ConfigNotFoundException, IOException, Js .withDestinationDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -178,8 +188,10 @@ void testGetDestinationVersionFromActorDefault() throws ConfigNotFoundException, when(mFeatureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(WORKSPACE_ID))).thenReturn(true); when(mConfigRepository.getDestinationConnection(ACTOR_ID)).thenReturn(new DestinationConnection().withDefaultVersionId(DEFAULT_VERSION_ID)); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -191,8 +203,10 @@ void testGetDestinationVersionWithOverride() throws ConfigNotFoundException, IOE .withDestinationDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(OVERRIDDEN_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(OVERRIDDEN_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertTrue(versionWithOverrideStatus.isOverrideApplied()); } @Test diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java index 1b9c203bef3..1cc3d9a4b3c 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java @@ -7,7 +7,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -21,6 +20,7 @@ import io.airbyte.config.ReleaseStage; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -39,36 +39,38 @@ */ class ActorDefinitionVersionPersistenceTest extends BaseConfigDatabaseTest { - private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); private static final String SOURCE_NAME = "Test Source"; private static final String DOCKER_REPOSITORY = "airbyte/source-test"; private static final String DOCKER_IMAGE_TAG = "0.1.0"; - private static final String DOCKER_IMAGE_TAG_2 = "0.2.0"; private static final String UNPERSISTED_DOCKER_IMAGE_TAG = "0.1.1"; private static final String PROTOCOL_VERSION = "1.0.0"; private static final ConnectorSpecification SPEC = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value"))).withProtocolVersion(PROTOCOL_VERSION); - private static final ConnectorSpecification SPEC_2 = new ConnectorSpecification() - .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value2"))).withProtocolVersion(PROTOCOL_VERSION); - private static final StandardSourceDefinition SOURCE_DEFINITION = new StandardSourceDefinition() - .withName(SOURCE_NAME) - .withSourceDefinitionId(ACTOR_DEFINITION_ID); + private static StandardSourceDefinition baseSourceDefinition(final UUID actorDefinitionId) { + return new StandardSourceDefinition() + .withName(SOURCE_NAME) + .withSourceDefinitionId(actorDefinitionId); + } - private static final ActorDefinitionVersion initialActorDefinitionVersion = new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID) - .withDockerImageTag("0.0.0") - .withDockerRepository("overwrite me") - .withSpec(new ConnectorSpecification().withAdditionalProperty("overwrite", "me").withProtocolVersion("0.0.0")); + private static ActorDefinitionVersion initialActorDefinitionVersion(final UUID actorDefinitionId) { + return new ActorDefinitionVersion() + .withActorDefinitionId(actorDefinitionId) + .withDockerImageTag("0.0.0") + .withDockerRepository("overwrite me") + .withSupportLevel(SupportLevel.COMMUNITY) + .withSpec(new ConnectorSpecification().withAdditionalProperty("overwrite", "me").withProtocolVersion("0.0.0")); + } - private static ActorDefinitionVersion baseActorDefinitionVersion() { + private static ActorDefinitionVersion baseActorDefinitionVersion(final UUID actorDefinitionId) { return new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID) + .withActorDefinitionId(actorDefinitionId) .withDockerRepository(DOCKER_REPOSITORY) .withDockerImageTag(DOCKER_IMAGE_TAG) .withSpec(SPEC) .withDocumentationUrl("https://airbyte.io/docs/") .withReleaseStage(ReleaseStage.BETA) + .withSupportLevel(SupportLevel.COMMUNITY) .withReleaseDate("2021-01-21") .withSuggestedStreams(new SuggestedStreams().withStreams(List.of("users"))) .withProtocolVersion("0.1.0") @@ -80,103 +82,109 @@ private static ActorDefinitionVersion baseActorDefinitionVersion() { .withNormalizationIntegrationType("bigquery")); } - private static final ActorDefinitionVersion ACTOR_DEFINITION_VERSION = baseActorDefinitionVersion(); - private ConfigRepository configRepository; + private StandardSourceDefinition sourceDefinition; @BeforeEach void beforeEach() throws Exception { truncateAllTables(); configRepository = new ConfigRepository(database, MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER); + final UUID defId = UUID.randomUUID(); + final ActorDefinitionVersion initialADV = initialActorDefinitionVersion(defId); + sourceDefinition = baseSourceDefinition(defId); + // Make sure that the source definition exists before we start writing actor definition versions - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, initialActorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, initialADV); } @Test void testWriteActorDefinitionVersion() throws IOException { - final ActorDefinitionVersion writtenADV = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion writtenADV = configRepository.writeActorDefinitionVersion(adv); + // All non-ID fields should match (the ID is randomly assigned) - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(writtenADV.getVersionId()), writtenADV); + final ActorDefinitionVersion expectedADV = adv.withVersionId(writtenADV.getVersionId()); + + assertEquals(expectedADV, writtenADV); } @Test void testGetActorDefinitionVersionByTag() throws IOException { - final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(adv); final UUID id = actorDefinitionVersion.getVersionId(); - final Optional optRetrievedADV = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); + final Optional optRetrievedADV = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optRetrievedADV.isPresent()); - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(id), optRetrievedADV.get()); + assertEquals(adv.withVersionId(id), optRetrievedADV.get()); } @Test void testGetForNonExistentTagReturnsEmptyOptional() throws IOException { - assertTrue(configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, UNPERSISTED_DOCKER_IMAGE_TAG).isEmpty()); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + assertTrue(configRepository.getActorDefinitionVersion(defId, UNPERSISTED_DOCKER_IMAGE_TAG).isEmpty()); } @Test void testGetActorDefinitionVersionById() throws IOException, ConfigNotFoundException { - final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(adv); final UUID id = actorDefinitionVersion.getVersionId(); final ActorDefinitionVersion retrievedADV = configRepository.getActorDefinitionVersion(id); assertNotNull(retrievedADV); - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(id), retrievedADV); + assertEquals(adv.withVersionId(id), retrievedADV); } @Test void testGetActorDefinitionVersionByIdNotExistentThrowsConfigNotFound() { - assertThrows(ConfigNotFoundException.class, () -> configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID)); + // Test using the definition id to catch any accidental assignment + final UUID defId = sourceDefinition.getSourceDefinitionId(); + + assertThrows(ConfigNotFoundException.class, () -> configRepository.getActorDefinitionVersion(defId)); } @Test - void testWriteSourceDefinitionAndDefaultVersion() throws IOException, JsonValidationException, ConfigNotFoundException { - // Write initial source definition and default version - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ACTOR_DEFINITION_VERSION); + void testWriteSourceDefinitionSupportLevelNone() throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId).withActorDefinitionId(defId).withSupportLevel(SupportLevel.NONE); - final Optional optADVForTag = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); + configRepository.writeConnectorMetadata(sourceDefinition, adv); + + final Optional optADVForTag = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optADVForTag.isPresent()); final ActorDefinitionVersion advForTag = optADVForTag.get(); - final StandardSourceDefinition retrievedSourceDefinition = - configRepository.getStandardSourceDefinition(SOURCE_DEFINITION.getSourceDefinitionId()); - assertEquals(retrievedSourceDefinition.getDefaultVersionId(), advForTag.getVersionId()); - - // Modify spec without changing docker image tag - final ActorDefinitionVersion modifiedADV = baseActorDefinitionVersion().withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, modifiedADV); - - assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(SOURCE_DEFINITION.getSourceDefinitionId())); - final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); - assertTrue(optADVForTagAfterCall2.isPresent()); - // Versioned data does not get updated since the tag did not change - old spec is still returned - assertEquals(advForTag, optADVForTagAfterCall2.get()); - - // Modifying docker image tag creates a new version (which can contain new versioned data) - final ActorDefinitionVersion newADV = baseActorDefinitionVersion().withDockerImageTag(DOCKER_IMAGE_TAG_2).withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, newADV); - - final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG_2); - assertTrue(optADVForTag2.isPresent()); - final ActorDefinitionVersion advForTag2 = optADVForTag2.get(); - - // Versioned data is updated as well as the version id - assertEquals(advForTag2, newADV.withVersionId(advForTag2.getVersionId())); - assertNotEquals(advForTag2.getVersionId(), advForTag.getVersionId()); - assertNotEquals(advForTag2.getSpec(), advForTag.getSpec()); + assertEquals(advForTag.getSupportLevel(), SupportLevel.NONE); + } + + @Test + void testWriteSourceDefinitionSupportLevelNonNullable() { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId).withActorDefinitionId(defId).withSupportLevel(null); + + assertThrows( + RuntimeException.class, + () -> configRepository.writeConnectorMetadata(sourceDefinition, adv)); } @Test void testAlwaysGetWithProtocolVersion() throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final List allActorDefVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("5.0.0").withProtocolVersion(null), - baseActorDefinitionVersion().withDockerImageTag("5.0.1") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.0").withProtocolVersion(null), + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.1") .withProtocolVersion(null) .withSpec(new ConnectorSpecification().withProtocolVersion("0.3.1")), - baseActorDefinitionVersion().withDockerImageTag("5.0.2") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.2") .withProtocolVersion("0.4.0") .withSpec(new ConnectorSpecification().withProtocolVersion("0.4.1")), - baseActorDefinitionVersion().withDockerImageTag("5.0.3") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.3") .withProtocolVersion("0.5.0") .withSpec(new ConnectorSpecification())); @@ -198,29 +206,31 @@ void testAlwaysGetWithProtocolVersion() throws IOException { @Test void testListActorDefinitionVersionsForDefinition() throws IOException, JsonValidationException, ConfigNotFoundException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); final StandardSourceDefinition otherSourceDef = new StandardSourceDefinition() .withName("Some other source") .withSourceDefinitionId(UUID.randomUUID()); - final ActorDefinitionVersion otherActorDefVersion = baseActorDefinitionVersion().withActorDefinitionId(otherSourceDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(otherSourceDef, otherActorDefVersion); + final ActorDefinitionVersion otherActorDefVersion = + baseActorDefinitionVersion(defId).withActorDefinitionId(otherSourceDef.getSourceDefinitionId()); + configRepository.writeConnectorMetadata(otherSourceDef, otherActorDefVersion); final UUID otherActorDefVersionId = configRepository.getStandardSourceDefinition(otherSourceDef.getSourceDefinitionId()).getDefaultVersionId(); final List actorDefinitionVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("1.0.0"), - baseActorDefinitionVersion().withDockerImageTag("2.0.0"), - baseActorDefinitionVersion().withDockerImageTag("3.0.0")); + baseActorDefinitionVersion(defId).withDockerImageTag("1.0.0"), + baseActorDefinitionVersion(defId).withDockerImageTag("2.0.0"), + baseActorDefinitionVersion(defId).withDockerImageTag("3.0.0")); final List expectedVersionIds = new ArrayList<>(); for (final ActorDefinitionVersion actorDefVersion : actorDefinitionVersions) { expectedVersionIds.add(configRepository.writeActorDefinitionVersion(actorDefVersion).getVersionId()); } - final UUID defaultVersionId = configRepository.getStandardSourceDefinition(ACTOR_DEFINITION_ID).getDefaultVersionId(); + final UUID defaultVersionId = configRepository.getStandardSourceDefinition(defId).getDefaultVersionId(); expectedVersionIds.add(defaultVersionId); final List actorDefinitionVersionsForDefinition = - configRepository.listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID); + configRepository.listActorDefinitionVersionsForDefinition(defId); assertThat(expectedVersionIds) .containsExactlyInAnyOrderElementsOf(actorDefinitionVersionsForDefinition.stream().map(ActorDefinitionVersion::getVersionId).toList()); assertFalse( @@ -237,12 +247,13 @@ void testListActorDefinitionVersionsForDefinition() throws IOException, JsonVali "UNSUPPORTED, DEPRECATED", }) void testSetActorDefinitionVersionSupportStates(final String initialSupportStateStr, final String targetSupportStateStr) throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); final SupportState initialSupportState = SupportState.valueOf(initialSupportStateStr); final SupportState targetSupportState = SupportState.valueOf(targetSupportStateStr); final List actorDefinitionVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("1.0.0").withSupportState(initialSupportState), - baseActorDefinitionVersion().withDockerImageTag("2.0.0").withSupportState(initialSupportState)); + baseActorDefinitionVersion(defId).withDockerImageTag("1.0.0").withSupportState(initialSupportState), + baseActorDefinitionVersion(defId).withDockerImageTag("2.0.0").withSupportState(initialSupportState)); final List versionIds = new ArrayList<>(); for (final ActorDefinitionVersion actorDefVersion : actorDefinitionVersions) { diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java index 5225fd2d646..6106dda28dc 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java @@ -9,7 +9,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; @@ -22,6 +21,7 @@ import java.sql.SQLException; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -42,10 +42,10 @@ void setup() throws SQLException, IOException, JsonValidationException { configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); standardSourceDefinition = MockData.publicSourceDefinition(); standardDestinationDefinition = MockData.publicDestinationDefinition(); - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, MockData.actorDefinitionVersion() + configRepository.writeConnectorMetadata(standardSourceDefinition, MockData.actorDefinitionVersion() .withActorDefinitionId(standardSourceDefinition.getSourceDefinitionId()) .withVersionId(standardSourceDefinition.getDefaultVersionId())); - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, MockData.actorDefinitionVersion() + configRepository.writeConnectorMetadata(standardDestinationDefinition, MockData.actorDefinitionVersion() .withActorDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) .withVersionId(standardDestinationDefinition.getDefaultVersionId())); configRepository.writeStandardWorkspaceNoSecrets(new StandardWorkspace() @@ -124,143 +124,99 @@ void testSetDestinationDefaultVersion() throws IOException, JsonValidationExcept } @Test - void testSourceDefaultVersionIsUpgradedOnNonbreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID sourceDefId = standardSourceDefinition.getSourceDefinitionId(); - final SourceConnection sourceConnection = new SourceConnection() + void testGetSourcesWithVersions() throws IOException { + final SourceConnection sourceConnection1 = new SourceConnection() .withSourceId(UUID.randomUUID()) - .withSourceDefinitionId(sourceDefId) + .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) .withWorkspaceId(WORKSPACE_ID) .withName(SOURCE_NAME); - configRepository.writeSourceConnectionNoSecrets(sourceConnection); + configRepository.writeSourceConnectionNoSecrets(sourceConnection1); - final UUID initialSourceDefinitionDefaultVersionId = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID initialSourceDefaultVersionId = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - assertNotNull(initialSourceDefinitionDefaultVersionId); - assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + final SourceConnection sourceConnection2 = new SourceConnection() + .withSourceId(UUID.randomUUID()) + .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) + .withWorkspaceId(WORKSPACE_ID) + .withName(SOURCE_NAME); + configRepository.writeSourceConnectionNoSecrets(sourceConnection2); - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(sourceDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); + final List sourceConnections = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId())); + assertEquals(2, sourceConnections.size()); + assertEquals( + Stream.of(sourceConnection1.getSourceId(), sourceConnection2.getSourceId()).sorted().toList(), + sourceConnections.stream().map(SourceConnection::getSourceId).sorted().toList()); - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, newVersion); - final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID sourceDefaultVersionIdAfterUpgrade = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + final ActorDefinitionVersion newActorDefinitionVersion = configRepository.writeActorDefinitionVersion(MockData.actorDefinitionVersion() + .withActorDefinitionId(standardSourceDefinition.getSourceDefinitionId()) + .withVersionId(UUID.randomUUID()) + .withDockerImageTag(UPGRADE_IMAGE_TAG)); + + configRepository.setActorDefaultVersion(sourceConnection1.getSourceId(), newActorDefinitionVersion.getVersionId()); + + final List sourcesWithNewVersion = + configRepository.listSourcesWithVersionIds(List.of(newActorDefinitionVersion.getVersionId())); + assertEquals(1, sourcesWithNewVersion.size()); + assertEquals(sourceConnection1.getSourceId(), sourcesWithNewVersion.get(0).getSourceId()); - assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(newVersionId, sourceDefaultVersionIdAfterUpgrade); + final List sourcesWithOldVersion = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId())); + assertEquals(1, sourcesWithOldVersion.size()); + assertEquals(sourceConnection2.getSourceId(), sourcesWithOldVersion.get(0).getSourceId()); + + final List sourcesWithBothVersions = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId(), newActorDefinitionVersion.getVersionId())); + assertEquals(2, sourcesWithBothVersions.size()); + assertEquals( + Stream.of(sourceConnection1.getSourceId(), sourceConnection2.getSourceId()).sorted().toList(), + sourcesWithBothVersions.stream().map(SourceConnection::getSourceId).sorted().toList()); } @Test - void testDestinationDefaultVersionIsUpgradedOnNonbreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID destinationDefId = standardDestinationDefinition.getDestinationDefinitionId(); - final DestinationConnection destinationConnection = new DestinationConnection() + void testGetDestinationsWithVersions() throws IOException { + final DestinationConnection destinationConnection1 = new DestinationConnection() .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(destinationDefId) + .withDestinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) .withWorkspaceId(WORKSPACE_ID) .withName(DESTINATION_NAME); - configRepository.writeDestinationConnectionNoSecrets(destinationConnection); - - final UUID initialDestinationDefinitionDefaultVersionId = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID initialDestinationDefaultVersionId = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - assertNotNull(initialDestinationDefinitionDefaultVersionId); - assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(destinationDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, newVersion); - final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID destinationDefaultVersionIdAfterUpgrade = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - - assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(newVersionId, destinationDefaultVersionIdAfterUpgrade); - } + configRepository.writeDestinationConnectionNoSecrets(destinationConnection1); - @Test - void testDestinationDefaultVersionIsNotModifiedOnBreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID destinationDefId = standardDestinationDefinition.getDestinationDefinitionId(); - final DestinationConnection destinationConnection = new DestinationConnection() + final DestinationConnection destinationConnection2 = new DestinationConnection() .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(destinationDefId) + .withDestinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) .withWorkspaceId(WORKSPACE_ID) .withName(DESTINATION_NAME); - configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection2); - final UUID initialDestinationDefinitionDefaultVersionId = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID initialDestinationDefaultVersionId = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - assertNotNull(initialDestinationDefinitionDefaultVersionId); - assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); - - // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking - final List breakingChangesForDef = - List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefId)); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(destinationDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, newVersion, breakingChangesForDef); - final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID destinationDefaultVersionIdAfterUpgrade = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - - assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(initialDestinationDefaultVersionId, destinationDefaultVersionIdAfterUpgrade); - } + final List destinationConnections = + configRepository.listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId())); + assertEquals(2, destinationConnections.size()); + assertEquals( + Stream.of(destinationConnection1.getDestinationId(), destinationConnection2.getDestinationId()).sorted().toList(), + destinationConnections.stream().map(DestinationConnection::getDestinationId).sorted().toList()); - @Test - void testSourceDefaultVersionIsNotModifiedOnBreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID sourceDefId = standardSourceDefinition.getSourceDefinitionId(); - final SourceConnection sourceConnection = new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withSourceDefinitionId(sourceDefId) - .withWorkspaceId(WORKSPACE_ID) - .withName(SOURCE_NAME); - configRepository.writeSourceConnectionNoSecrets(sourceConnection); + final ActorDefinitionVersion newActorDefinitionVersion = configRepository.writeActorDefinitionVersion(MockData.actorDefinitionVersion() + .withActorDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) + .withVersionId(UUID.randomUUID()) + .withDockerImageTag(UPGRADE_IMAGE_TAG)); + + configRepository.setActorDefaultVersion(destinationConnection1.getDestinationId(), newActorDefinitionVersion.getVersionId()); + + final List destinationsWithNewVersion = + configRepository.listDestinationsWithVersionIds(List.of(newActorDefinitionVersion.getVersionId())); + assertEquals(1, destinationsWithNewVersion.size()); + assertEquals(destinationConnection1.getDestinationId(), destinationsWithNewVersion.get(0).getDestinationId()); + + final List destinationsWithOldVersion = + configRepository.listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId())); + assertEquals(1, destinationsWithOldVersion.size()); + assertEquals(destinationConnection2.getDestinationId(), destinationsWithOldVersion.get(0).getDestinationId()); - final UUID initialSourceDefinitionDefaultVersionId = - configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID initialSourceDefaultVersionId = - configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - assertNotNull(initialSourceDefinitionDefaultVersionId); - assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); - - // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking - final List breakingChangesForDef = - List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefId)); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(sourceDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, newVersion, breakingChangesForDef); - final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID sourceDefaultVersionIdAfterUpgrade = - configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - - assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterUpgrade); + final List destinationsWithBothVersions = configRepository + .listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId(), newActorDefinitionVersion.getVersionId())); + assertEquals(2, destinationsWithBothVersions.size()); + assertEquals( + Stream.of(destinationConnection1.getDestinationId(), destinationConnection2.getDestinationId()).sorted().toList(), + destinationsWithBothVersions.stream().map(DestinationConnection::getDestinationId).sorted().toList()); } } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java index d6159cea25b..70d4d5ecfd3 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @@ -130,6 +130,9 @@ static void truncateAllTables() throws SQLException { state, stream_reset, \"user\", + user_invitation, + sso_config, + organization_email_domain, workspace, workspace_service_account """)); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java index 29dcc65ef8f..ef94c6dea87 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java @@ -15,6 +15,7 @@ import io.airbyte.config.ActorDefinitionConfigInjection; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; import java.util.Map; @@ -103,7 +104,7 @@ void testCreate() throws IOException { private void createBaseObjects() throws IOException { sourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); createInjection(sourceDefinition, "a"); createInjection(sourceDefinition, "b"); @@ -111,7 +112,7 @@ private void createBaseObjects() throws IOException { // unreachable injection, should not show up final StandardSourceDefinition otherSourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(otherSourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(otherSourceDefinition, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(otherSourceDefinition, actorDefinitionVersion2); createInjection(otherSourceDefinition, "c"); } @@ -139,6 +140,7 @@ private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actor .withActorDefinitionId(actorDefId) .withDockerRepository("source-image-" + actorDefId) .withDockerImageTag("1.0.0") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 960db980cd0..f66fe4c188f 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -12,6 +12,7 @@ import static org.jooq.impl.DSL.select; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.spy; @@ -83,13 +84,13 @@ void setup() throws IOException, JsonValidationException, SQLException { final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); } for (final StandardDestinationDefinition destinationDefinition : MockData.standardDestinationDefinitions()) { final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(destinationDefinition.getDestinationDefinitionId()) .withVersionId(destinationDefinition.getDefaultVersionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); } for (final SourceConnection source : MockData.sourceConnections()) { configRepository.writeSourceConnectionNoSecrets(source); @@ -156,7 +157,7 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final SourceConnection source = new SourceConnection() .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) @@ -169,12 +170,12 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce final AirbyteCatalog firstCatalog = CatalogHelpers.createAirbyteCatalog("product", Field.of("label", JsonSchemaType.STRING), Field.of("size", JsonSchemaType.NUMBER), Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); - configRepository.writeCanonicalActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); + configRepository.writeActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); final AirbyteCatalog secondCatalog = CatalogHelpers.createAirbyteCatalog("product", Field.of("size", JsonSchemaType.NUMBER), Field.of("label", JsonSchemaType.STRING), Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); - configRepository.writeCanonicalActorCatalogFetchEvent(secondCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, otherConfigHash); + configRepository.writeActorCatalogFetchEvent(secondCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, otherConfigHash); final String expectedCatalog = "{" @@ -202,6 +203,60 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce assertEquals(expectedCatalog, Jsons.serialize(catalogResult.get().getCatalog())); } + @Test + void testWriteCanonicalHashActorCatalog() throws IOException, JsonValidationException, SQLException { + final String canonicalConfigHash = "8ad32981"; + final StandardWorkspace workspace = MockData.standardWorkspaces().get(0); + + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withSourceType(SourceType.DATABASE) + .withName("sourceDefinition"); + final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) + .withVersionId(sourceDefinition.getDefaultVersionId()); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); + + final SourceConnection source = new SourceConnection() + .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) + .withSourceId(UUID.randomUUID()) + .withName("SomeConnector") + .withWorkspaceId(workspace.getWorkspaceId()) + .withConfiguration(Jsons.deserialize("{}")); + configRepository.writeSourceConnectionNoSecrets(source); + + final AirbyteCatalog firstCatalog = CatalogHelpers.createAirbyteCatalog("product", + Field.of("label", JsonSchemaType.STRING), Field.of("size", JsonSchemaType.NUMBER), + Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); + configRepository.writeActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); + + String expectedCatalog = + "{" + + "\"streams\":[" + + "{" + + "\"default_cursor_field\":[]," + + "\"json_schema\":{" + + "\"properties\":{" + + "\"color\":{\"type\":\"string\"}," + + "\"label\":{\"type\":\"string\"}," + + "\"price\":{\"type\":\"number\"}," + + "\"size\":{\"type\":\"number\"}" + + "}," + + "\"type\":\"object\"" + + "}," + + "\"name\":\"product\"," + + "\"source_defined_primary_key\":[]," + + "\"supported_sync_modes\":[\"full_refresh\"]" + + "}" + + "]" + + "}"; + + final Optional catalogResult = configRepository.getActorCatalog(source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); + assertTrue(catalogResult.isPresent()); + assertEquals(catalogResult.get().getCatalogHash(), canonicalConfigHash); + assertEquals(expectedCatalog, Jsons.canonicalJsonSerialize(catalogResult.get().getCatalog())); + } + @Test void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, SQLException { final String otherConfigHash = "OtherConfigHash"; @@ -214,7 +269,7 @@ void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final SourceConnection source = new SourceConnection() .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) @@ -809,4 +864,12 @@ private static void writeActorCatalog(final List configs, final DS }); } + @Test + void testGetEarlySyncJobs() throws IOException { + // This test just verifies that the query can be run against configAPI DB. + // The query has been tested locally against prod DB to verify the outputs. + final Set earlySyncJobs = configRepository.listEarlySyncJobs(7, 30); + assertNotNull(earlySyncJobs); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java index 8412fc19e33..3ed27eafca1 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java @@ -19,6 +19,7 @@ import io.airbyte.config.DeclarativeManifest; import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Jsons; import java.io.IOException; @@ -160,7 +161,7 @@ void testUpdate() throws IOException, ConfigNotFoundException { void whenUpdateBuilderProjectAndActorDefinitionThenUpdateConnectorBuilderAndActorDefinition() throws Exception { configRepository.writeBuilderProjectDraft(A_BUILDER_PROJECT_ID, A_WORKSPACE_ID, A_PROJECT_NAME, A_MANIFEST); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(A_WORKSPACE_ID)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(MockData.customSourceDefinition() + configRepository.writeCustomConnectorMetadata(MockData.customSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_PROJECT_NAME) .withPublic(false), @@ -179,7 +180,7 @@ void whenUpdateBuilderProjectAndActorDefinitionThenUpdateConnectorBuilderAndActo void givenSourceIsPublicWhenUpdateBuilderProjectAndActorDefinitionThenActorDefinitionNameIsNotUpdated() throws Exception { configRepository.writeBuilderProjectDraft(A_BUILDER_PROJECT_ID, A_WORKSPACE_ID, A_PROJECT_NAME, A_MANIFEST); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(A_WORKSPACE_ID)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(MockData.customSourceDefinition() + configRepository.writeCustomConnectorMetadata(MockData.customSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_PROJECT_NAME) .withPublic(true), @@ -321,9 +322,10 @@ private StandardSourceDefinition linkSourceDefinition(final UUID projectId) thro .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withDockerRepository("repo-" + id) .withDockerImageTag("0.0.1") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); configRepository.insertActiveDeclarativeManifest(new DeclarativeManifest() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersion(MANIFEST_VERSION) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java new file mode 100644 index 00000000000..a458b582943 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java @@ -0,0 +1,529 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Geography; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Tests for configRepository methods that write connector metadata together. Includes writing + * global metadata (source/destination definitions and breaking changes) and versioned metadata + * (actor definition versions) as well as the logic that determines whether actors should be + * upgraded upon changing the dockerImageTag. + */ +class ConnectorMetadataPersistenceTest extends BaseConfigDatabaseTest { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + + private static final String DOCKER_IMAGE_TAG = "0.0.1"; + + private static final String UPGRADE_IMAGE_TAG = "0.0.2"; + private static final String PROTOCOL_VERSION = "1.0.0"; + + private ConfigRepository configRepository; + + @BeforeEach + void setup() throws SQLException, JsonValidationException, IOException { + truncateAllTables(); + + configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); + configRepository.writeStandardWorkspaceNoSecrets(new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withName("default") + .withSlug("workspace-slug") + .withInitialSetupComplete(false) + .withTombstone(false) + .withDefaultGeography(Geography.US)); + } + + @Test + void testWriteConnectorMetadataForSource() + throws JsonValidationException, IOException, ConfigNotFoundException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + final Optional actorDefinitionVersionFromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); + + assertTrue(actorDefinitionVersionFromDB.isPresent()); + final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); + + assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); + assertEquals(firstVersionId, sourceDefinitionFromDB.getDefaultVersionId()); + assertEquals(sourceDefinition.withDefaultVersionId(firstVersionId), sourceDefinitionFromDB); + + // Updating an existing source definition/version + final StandardSourceDefinition sourceDefinition2 = sourceDefinition.withName("updated name"); + final ActorDefinitionVersion actorDefinitionVersion2 = + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(UPGRADE_IMAGE_TAG); + final List breakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefinition2.getSourceDefinitionId())); + configRepository.writeConnectorMetadata(sourceDefinition2, actorDefinitionVersion2, breakingChanges); + + sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + final Optional actorDefinitionVersion2FromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); + final List breakingChangesForDefFromDb = + configRepository.listBreakingChangesForActorDefinition(sourceDefinition2.getSourceDefinitionId()); + + assertTrue(actorDefinitionVersion2FromDB.isPresent()); + final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); + + assertNotEquals(firstVersionId, newADVId); + assertEquals(newADVId, sourceDefinitionFromDB.getDefaultVersionId()); + assertEquals(sourceDefinition2.withDefaultVersionId(newADVId), sourceDefinitionFromDB); + assertThat(breakingChangesForDefFromDb).containsExactlyInAnyOrderElementsOf(breakingChanges); + } + + @Test + void testWriteConnectorMetadataForDestination() + throws JsonValidationException, IOException, ConfigNotFoundException { + // Initial insert + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + StandardDestinationDefinition destinationDefinitionFromDB = + configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); + final Optional actorDefinitionVersionFromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); + + assertTrue(actorDefinitionVersionFromDB.isPresent()); + final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); + + assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); + assertEquals(firstVersionId, destinationDefinitionFromDB.getDefaultVersionId()); + assertEquals(destinationDefinition.withDefaultVersionId(firstVersionId), destinationDefinitionFromDB); + + // Updating an existing destination definition/version + final StandardDestinationDefinition destinationDefinition2 = destinationDefinition.withName("updated name"); + final ActorDefinitionVersion actorDefinitionVersion2 = + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(UPGRADE_IMAGE_TAG); + final List breakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefinition2.getDestinationDefinitionId())); + configRepository.writeConnectorMetadata(destinationDefinition2, actorDefinitionVersion2, breakingChanges); + + destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); + final Optional actorDefinitionVersion2FromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); + final List breakingChangesForDefFromDb = + configRepository.listBreakingChangesForActorDefinition(destinationDefinition2.getDestinationDefinitionId()); + + assertTrue(actorDefinitionVersion2FromDB.isPresent()); + final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); + + assertNotEquals(firstVersionId, newADVId); + assertEquals(newADVId, destinationDefinitionFromDB.getDefaultVersionId()); + assertEquals(destinationDefinition2.withDefaultVersionId(newADVId), destinationDefinitionFromDB); + assertThat(breakingChangesForDefFromDb).containsExactlyInAnyOrderElementsOf(breakingChanges); + } + + @Test + void testVersionedFieldsDoNotChangeWithoutVersionBump() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final UUID actorDefinitionId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(actorDefinitionId); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final Optional optADVForTag = configRepository.getActorDefinitionVersion(actorDefinitionId, DOCKER_IMAGE_TAG); + assertTrue(optADVForTag.isPresent()); + final ActorDefinitionVersion advForTag = optADVForTag.get(); + final StandardSourceDefinition retrievedSourceDefinition = + configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + assertEquals(retrievedSourceDefinition.getDefaultVersionId(), advForTag.getVersionId()); + + final ConnectorSpecification updatedSpec = new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value2"))).withProtocolVersion(PROTOCOL_VERSION); + + // Modify spec without changing docker image tag + final ActorDefinitionVersion modifiedADV = createBaseActorDefVersion(actorDefinitionId).withSpec(updatedSpec); + configRepository.writeConnectorMetadata(sourceDefinition, modifiedADV); + + assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())); + final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(actorDefinitionId, DOCKER_IMAGE_TAG); + assertTrue(optADVForTagAfterCall2.isPresent()); + // Versioned data does not get updated since the tag did not change - old spec is still returned + assertEquals(advForTag, optADVForTagAfterCall2.get()); + + // Modifying docker image tag creates a new version (which can contain new versioned data) + final ActorDefinitionVersion newADV = createBaseActorDefVersion(actorDefinitionId).withDockerImageTag(UPGRADE_IMAGE_TAG).withSpec(updatedSpec); + configRepository.writeConnectorMetadata(sourceDefinition, newADV); + + final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(actorDefinitionId, UPGRADE_IMAGE_TAG); + assertTrue(optADVForTag2.isPresent()); + final ActorDefinitionVersion advForTag2 = optADVForTag2.get(); + + // Versioned data is updated as well as the version id + assertEquals(advForTag2, newADV.withVersionId(advForTag2.getVersionId())); + assertNotEquals(advForTag2.getVersionId(), advForTag.getVersionId()); + assertNotEquals(advForTag2.getSpec(), advForTag.getSpec()); + } + + @ParameterizedTest + @ValueSource(strings = {"2.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) + void testCustomImageTagsDoNotBreakCustomConnectorUpgrade(final String dockerImageTag) throws IOException { + // Initial insert + final StandardSourceDefinition customSourceDefinition = createBaseSourceDef().withCustom(true); + final StandardDestinationDefinition customDestinationDefinition = createBaseDestDef().withCustom(true); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = + createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(customSourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(customDestinationDefinition, destinationActorDefinitionVersion); + + // Update + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(customSourceDefinition, + createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(customDestinationDefinition, + createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + } + + @ParameterizedTest + @ValueSource(strings = {"2.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) + void testImageTagExpectationsNorNonCustomConnectorUpgradesWithoutBreakingChanges(final String dockerImageTag) throws IOException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(sourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, destinationActorDefinitionVersion); + + // Update + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + } + + @ParameterizedTest + @CsvSource({"0.0.1, true", "dev, true", "test, false", "1.9.1-dev.33a53e6236, true", "97b69a76-1f06-4680-8905-8beda74311d0, false"}) + void testImageTagExpectationsNorNonCustomConnectorUpgradesWithBreakingChanges(final String dockerImageTag, final boolean upgradeShouldSucceed) + throws IOException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(sourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, destinationActorDefinitionVersion); + + final List sourceBreakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefinition.getSourceDefinitionId())); + final List destinationBreakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefinition.getDestinationDefinitionId())); + + // Update + if (upgradeShouldSucceed) { + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), + destinationBreakingChanges)); + } else { + assertThrows(IllegalArgumentException.class, () -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); + assertThrows(IllegalArgumentException.class, () -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), + destinationBreakingChanges)); + } + } + + @Test + void testSourceDefaultVersionIsUpgradedOnNonbreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(sourceDefinition, newVersion); + final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterUpgrade = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + + assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(newVersionId, sourceDefaultVersionIdAfterUpgrade); + } + + @Test + void testDestinationDefaultVersionIsUpgradedOnNonbreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + final UUID destinationDefId = destinationDefinition.getDestinationDefinitionId(); + final DestinationConnection destinationConnection = createBaseDestinationActor(destinationDefId); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + + final UUID initialDestinationDefinitionDefaultVersionId = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID initialDestinationDefaultVersionId = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + assertNotNull(initialDestinationDefinitionDefaultVersionId); + assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(destinationDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(destinationDefinition, newVersion); + final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID destinationDefaultVersionIdAfterUpgrade = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + + assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(newVersionId, destinationDefaultVersionIdAfterUpgrade); + } + + @Test + void testDestinationDefaultVersionIsNotModifiedOnBreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + final UUID destinationDefId = destinationDefinition.getDestinationDefinitionId(); + final DestinationConnection destinationConnection = createBaseDestinationActor(destinationDefId); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + + final UUID initialDestinationDefinitionDefaultVersionId = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID initialDestinationDefaultVersionId = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + assertNotNull(initialDestinationDefinitionDefaultVersionId); + assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(destinationDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(destinationDefinition, newVersion, breakingChangesForDef); + final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID destinationDefaultVersionIdAfterUpgrade = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + + assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(initialDestinationDefaultVersionId, destinationDefaultVersionIdAfterUpgrade); + } + + @Test + void testSourceDefaultVersionIsNotModifiedOnBreakingUpgrade() + throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(sourceDefinition, newVersion, breakingChangesForDef); + final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterUpgrade = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + + assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterUpgrade); + } + + @Test + void testTransactionRollbackOnFailure() throws IOException, JsonValidationException, ConfigNotFoundException { + final UUID initialADVId = UUID.randomUUID(); + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withVersionId(initialADVId); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking, but + // with a tag that will + // fail validation. We want to check that the state is rolled back correctly. + final String invalidUpgradeTag = "1.0"; + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange("1.0.0").withActorDefinitionId(sourceDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(invalidUpgradeTag) + .withDocumentationUrl("https://www.something.new"); + + final StandardSourceDefinition updatedSourceDefinition = Jsons.clone(sourceDefinition).withName("updated name"); + + assertThrows(IllegalArgumentException.class, + () -> configRepository.writeConnectorMetadata(updatedSourceDefinition, newVersion, breakingChangesForDef)); + + final UUID sourceDefinitionDefaultVersionIdAfterFailedUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterFailedUpgrade = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + final StandardSourceDefinition sourceDefinitionAfterFailedUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId); + final Optional newActorDefinitionVersionAfterFailedUpgrade = + configRepository.getActorDefinitionVersion(sourceDefId, invalidUpgradeTag); + final ActorDefinitionVersion defaultActorDefinitionVersionAfterFailedUpgrade = + configRepository.getActorDefinitionVersion(sourceDefinitionDefaultVersionIdAfterFailedUpgrade); + + // New actor definition version was not persisted + assertFalse(newActorDefinitionVersionAfterFailedUpgrade.isPresent()); + // Valid breaking change was not persisted + assertEquals(0, configRepository.listBreakingChangesForActorDefinition(sourceDefId).size()); + + // Neither the default version nor the actors get upgraded, the actors are still on the default + // version + assertEquals(initialSourceDefaultVersionId, sourceDefinitionDefaultVersionIdAfterFailedUpgrade); + assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterFailedUpgrade); + + // Source definition metadata is the same as before + assertEquals(sourceDefinition.withDefaultVersionId(initialADVId), sourceDefinitionAfterFailedUpgrade); + // Actor definition metadata is the same as before + assertEquals(actorDefinitionVersion1, defaultActorDefinitionVersionAfterFailedUpgrade); + } + + private static StandardSourceDefinition createBaseSourceDef() { + final UUID id = UUID.randomUUID(); + + return new StandardSourceDefinition() + .withName("source-def-" + id) + .withSourceDefinitionId(id) + .withTombstone(false) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actorDefId) { + return new ActorDefinitionVersion() + .withVersionId(UUID.randomUUID()) + .withActorDefinitionId(actorDefId) + .withDockerRepository("source-image-" + actorDefId) + .withDockerImageTag(DOCKER_IMAGE_TAG) + .withProtocolVersion(PROTOCOL_VERSION) + .withSupportLevel(SupportLevel.CERTIFIED) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value1"))).withProtocolVersion(PROTOCOL_VERSION)); + } + + private static StandardDestinationDefinition createBaseDestDef() { + final UUID id = UUID.randomUUID(); + + return new StandardDestinationDefinition() + .withName("source-def-" + id) + .withDestinationDefinitionId(id) + .withTombstone(false); + } + + private static SourceConnection createBaseSourceActor(final UUID actorDefinitionId) { + final UUID id = UUID.randomUUID(); + + return new SourceConnection() + .withSourceId(id) + .withSourceDefinitionId(actorDefinitionId) + .withWorkspaceId(WORKSPACE_ID) + .withName("source-" + id); + } + + private static DestinationConnection createBaseDestinationActor(final UUID actorDefinitionId) { + final UUID id = UUID.randomUUID(); + + return new DestinationConnection() + .withDestinationId(id) + .withDestinationDefinitionId(actorDefinitionId) + .withWorkspaceId(WORKSPACE_ID) + .withName("destination-" + id); + } + +} diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java index bd692ce260a..d0afcc88e06 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java @@ -248,7 +248,7 @@ void givenActiveDeclarativeManifestWithActorDefinitionId(final UUID actorDefinit void givenSourceDefinition(final UUID sourceDefinitionId) throws JsonValidationException, IOException { final UUID workspaceId = UUID.randomUUID(); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(workspaceId)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion( + configRepository.writeCustomConnectorMetadata( MockData.customSourceDefinition().withSourceDefinitionId(sourceDefinitionId), MockData.actorDefinitionVersion().withActorDefinitionId(sourceDefinitionId), workspaceId, diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 65f577a1fb6..7b86503e263 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -49,6 +49,7 @@ import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.State; +import io.airbyte.config.SupportLevel; import io.airbyte.config.User; import io.airbyte.config.User.AuthProvider; import io.airbyte.config.WebhookConfig; @@ -131,6 +132,8 @@ public class MockData { static final UUID CREATOR_USER_ID_2 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_3 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_4 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_5 = UUID.randomUUID(); + // Permission static final UUID PERMISSION_ID_1 = UUID.randomUUID(); static final UUID PERMISSION_ID_2 = UUID.randomUUID(); @@ -138,6 +141,8 @@ public class MockData { static final UUID PERMISSION_ID_4 = UUID.randomUUID(); static final UUID PERMISSION_ID_5 = UUID.randomUUID(); + static final UUID PERMISSION_ID_6 = UUID.randomUUID(); + static final UUID PERMISSION_ID_7 = UUID.randomUUID(); static final UUID ORGANIZATION_ID_1 = UUID.randomUUID(); static final UUID ORGANIZATION_ID_2 = UUID.randomUUID(); @@ -229,7 +234,18 @@ public static List users() { .withEmail("user-4@whatever.com") .withNews(true); - return Arrays.asList(user1, user2, user3, user4); + final User user5 = new User() + .withUserId(CREATOR_USER_ID_5) + .withName("user-5") + .withAuthUserId(CREATOR_USER_ID_5.toString()) + .withAuthProvider(AuthProvider.KEYCLOAK) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("company-5") + .withEmail("user-5@whatever.com") + .withNews(true); + + return Arrays.asList(user1, user2, user3, user4, user5); } public static List permissions() { @@ -263,7 +279,19 @@ public static List permissions() { .withOrganizationId(ORGANIZATION_ID_1) .withPermissionType(PermissionType.ORGANIZATION_ADMIN); - return Arrays.asList(permission1, permission2, permission3, permission4, permission5); + final Permission permission6 = new Permission() + .withPermissionId(PERMISSION_ID_6) + .withUserId(CREATOR_USER_ID_5) + .withWorkspaceId(WORKSPACE_ID_2) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission7 = new Permission() + .withPermissionId(PERMISSION_ID_7) + .withUserId(CREATOR_USER_ID_5) + .withOrganizationId(ORGANIZATION_ID_2) + .withPermissionType(PermissionType.ORGANIZATION_READER); + + return Arrays.asList(permission1, permission2, permission3, permission4, permission5, permission6, permission7); } public static List organizations() { @@ -379,6 +407,7 @@ public static ActorDefinitionVersion actorDefinitionVersion() { .withDockerImageTag("0.0.1") .withDockerRepository("repository-4") .withSpec(connectorSpecification()) + .withSupportLevel(SupportLevel.COMMUNITY) .withProtocolVersion("0.2.0"); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java index c2bcff00396..bcf6246dd16 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java @@ -26,6 +26,19 @@ void beforeEach() throws Exception { } } + @Test + void createOrganization() throws Exception { + Organization organization = new Organization() + .withOrganizationId(UUID.randomUUID()) + .withUserId(UUID.randomUUID()) + .withEmail("octavia@airbyte.io") + .withName("new org"); + organizationPersistence.createOrganization(organization); + Optional result = organizationPersistence.getOrganization(organization.getOrganizationId()); + assertTrue(result.isPresent()); + assertEquals(organization, result.get()); + } + @Test void getOrganization() throws Exception { Optional result = organizationPersistence.getOrganization(MockData.ORGANIZATION_ID_1); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java index 8725e533294..cefe562bf11 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java @@ -6,8 +6,10 @@ import io.airbyte.config.Organization; import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.UserPermission; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -102,4 +104,26 @@ void listUsersInWorkspaceTest() throws IOException { Assertions.assertEquals(2, userPermissions.size()); } + @Test + void listInstanceUsersTest() throws IOException { + final List userPermissions = permissionPersistence.listInstanceAdminUsers(); + Assertions.assertEquals(1, userPermissions.size()); + UserPermission userPermission = userPermissions.get(0); + Assertions.assertEquals(MockData.CREATOR_USER_ID_1, userPermission.getUser().getUserId()); + } + + @Test + void findUsersInWorkspaceTest() throws Exception { + final PermissionType permissionType = permissionPersistence + .findPermissionTypeForUserAndWorkspace(MockData.WORKSPACE_ID_2, MockData.CREATOR_USER_ID_5.toString(), AuthProvider.KEYCLOAK); + Assertions.assertEquals(PermissionType.WORKSPACE_ADMIN, permissionType); + } + + @Test + void findUsersInOrganizationTest() throws Exception { + final PermissionType permissionType = permissionPersistence + .findPermissionTypeForUserAndOrganization(MockData.ORGANIZATION_ID_2, MockData.CREATOR_USER_ID_5.toString(), AuthProvider.KEYCLOAK); + Assertions.assertEquals(PermissionType.ORGANIZATION_READER, permissionType); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java index a54bd41f281..aec78fe6c54 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java @@ -32,6 +32,7 @@ import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.enums.AutoPropagationStatus; import io.airbyte.db.instance.configs.jooq.generated.enums.NotificationType; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; @@ -355,8 +356,9 @@ private StandardSourceDefinition createStandardSourceDefinition(final String pro .withDocumentationUrl("documentation-url-1") .withSpec(new ConnectorSpecification()) .withProtocolVersion(protocolVersion) - .withReleaseStage(releaseStage); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef, sourceDefVersion); + .withReleaseStage(releaseStage) + .withSupportLevel(SupportLevel.COMMUNITY); + configRepository.writeConnectorMetadata(sourceDef, sourceDefVersion); return sourceDef; } @@ -378,8 +380,10 @@ private StandardDestinationDefinition createStandardDestDefinition(final String .withDocumentationUrl("documentation-url-3") .withSpec(new ConnectorSpecification()) .withProtocolVersion(protocolVersion) - .withReleaseStage(releaseStage); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, destDefVersion); + .withReleaseStage(releaseStage) + .withSupportLevel(SupportLevel.COMMUNITY); + + configRepository.writeConnectorMetadata(destDef, destDefVersion); return destDef; } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java index d510de40044..e1966578ad9 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java @@ -74,9 +74,9 @@ private void setupTestData() throws JsonValidationException, IOException { final StandardSync sync = Jsons.clone(MockData.standardSyncs().get(0)).withOperationIds(Collections.emptyList()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(sourceConnection); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion2); configRepository.writeDestinationConnectionNoSecrets(destinationConnection); configRepository.writeStandardSync(sync); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java index d0562d1f097..3a607a6c21c 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java @@ -4,7 +4,10 @@ package io.airbyte.config.persistence; +import static io.airbyte.featureflag.ContextKt.ANONYMOUS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -15,32 +18,56 @@ import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; +import io.airbyte.config.Configs.DeploymentMode; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.config.persistence.SupportStateUpdater.SupportStateUpdate; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.PauseSyncsWithUnsupportedActors; +import io.airbyte.featureflag.TestClient; +import io.airbyte.featureflag.Workspace; +import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.time.LocalDate; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SupportStateUpdaterTest { private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_ID_2 = UUID.randomUUID(); private static final String V0_1_0 = "0.1.0"; private static final String V1_0_0 = "1.0.0"; private static final String V1_1_0 = "1.1.0"; private static final String V2_0_0 = "2.0.0"; + private static final String V3_0_0 = "3.0.0"; private ConfigRepository mConfigRepository; + private ActorDefinitionVersionHelper mActorDefinitionVersionHelper; + private FeatureFlagClient mFeatureFlagClient; + private SupportStateUpdater supportStateUpdater; @BeforeEach void setup() { mConfigRepository = mock(ConfigRepository.class); - supportStateUpdater = new SupportStateUpdater(mConfigRepository); + mActorDefinitionVersionHelper = mock(ActorDefinitionVersionHelper.class); + mFeatureFlagClient = mock(TestClient.class); + supportStateUpdater = new SupportStateUpdater(mConfigRepository, mActorDefinitionVersionHelper, DeploymentMode.CLOUD, mFeatureFlagClient); + + when(mFeatureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) + .thenReturn(true); } ActorDefinitionBreakingChange createBreakingChange(final String version, final String upgradeDeadline) { @@ -61,6 +88,24 @@ ActorDefinitionVersion createActorDefinitionVersion(final String version) { .withSupportState(null); // clear support state to always need a SupportStateUpdate. } + @Test + void testShouldNotDisableSyncsInOSS() { + assertTrue(supportStateUpdater.shouldDisableSyncs()); + + supportStateUpdater = new SupportStateUpdater(mConfigRepository, mActorDefinitionVersionHelper, DeploymentMode.OSS, mFeatureFlagClient); + assertFalse(supportStateUpdater.shouldDisableSyncs()); + } + + @Test + void testShouldNotDisableSyncsWhenFFDisabled() { + assertTrue(supportStateUpdater.shouldDisableSyncs()); + + when(mFeatureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) + .thenReturn(false); + + assertFalse(supportStateUpdater.shouldDisableSyncs()); + } + @Test void testUpdateSupportStatesForCustomDestinationDefinitionNoOp() throws ConfigNotFoundException, IOException { supportStateUpdater.updateSupportStatesForDestinationDefinition(new StandardDestinationDefinition().withCustom(true)); @@ -98,6 +143,7 @@ void testUpdateSupportStatesForDestinationDefinition() throws ConfigNotFoundExce verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_0_1_0.getVersionId()), SupportState.UNSUPPORTED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test @@ -125,10 +171,11 @@ void testUpdateSupportStatesForSourceDefinition() throws ConfigNotFoundException verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_0_1_0.getVersionId()), SupportState.UNSUPPORTED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test - void testUpdateSupportStates() throws IOException { + void testUpdateSupportStates() throws IOException, JsonValidationException, ConfigNotFoundException { final ActorDefinitionVersion SRC_V0_1_0 = createActorDefinitionVersion(V0_1_0); final ActorDefinitionVersion SRC_V1_0_0 = createActorDefinitionVersion(V1_0_0); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() @@ -137,39 +184,94 @@ void testUpdateSupportStates() throws IOException { .withDefaultVersionId(SRC_V1_0_0.getVersionId()); final UUID destinationDefinitionId = UUID.randomUUID(); - final ActorDefinitionVersion DEST_V1_1_0 = createActorDefinitionVersion(V1_1_0) + final ActorDefinitionVersion DEST_V0_1_0 = createActorDefinitionVersion(V0_1_0) .withActorDefinitionId(destinationDefinitionId); - final ActorDefinitionVersion DEST_V2_0_0 = createActorDefinitionVersion(V2_0_0) + final ActorDefinitionVersion DEST_V1_0_0 = createActorDefinitionVersion(V1_0_0) .withActorDefinitionId(destinationDefinitionId); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() .withName("destination") .withDestinationDefinitionId(destinationDefinitionId) - .withDefaultVersionId(DEST_V2_0_0.getVersionId()); + .withDefaultVersionId(DEST_V1_0_0.getVersionId()); final ActorDefinitionBreakingChange SRC_BC_1_0_0 = createBreakingChange(V1_0_0, "2020-01-01"); - final ActorDefinitionBreakingChange DEST_BC_2_0_0 = createBreakingChange(V2_0_0, "2020-02-01") + final ActorDefinitionBreakingChange DEST_BC_1_0_0 = createBreakingChange(V1_0_0, "2020-02-01") .withActorDefinitionId(destinationDefinitionId); when(mConfigRepository.listPublicSourceDefinitions(false)).thenReturn(List.of(sourceDefinition)); when(mConfigRepository.listPublicDestinationDefinitions(false)).thenReturn(List.of(destinationDefinition)); - when(mConfigRepository.listBreakingChanges()).thenReturn(List.of(SRC_BC_1_0_0, DEST_BC_2_0_0)); + when(mConfigRepository.listBreakingChanges()).thenReturn(List.of(SRC_BC_1_0_0, DEST_BC_1_0_0)); when(mConfigRepository.listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID)) .thenReturn(List.of(SRC_V0_1_0, SRC_V1_0_0)); when(mConfigRepository.listActorDefinitionVersionsForDefinition(destinationDefinitionId)) - .thenReturn(List.of(DEST_V1_1_0, DEST_V2_0_0)); + .thenReturn(List.of(DEST_V0_1_0, DEST_V1_0_0)); + when(mConfigRepository.listSourcesWithVersionIds(List.of(SRC_V0_1_0.getVersionId()))).thenReturn(List.of()); - supportStateUpdater.updateSupportStates(); + supportStateUpdater.updateSupportStates(LocalDate.parse("2020-01-15")); verify(mConfigRepository).listPublicSourceDefinitions(false); verify(mConfigRepository).listPublicDestinationDefinitions(false); verify(mConfigRepository).listBreakingChanges(); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(destinationDefinitionId); - verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V0_1_0.getVersionId(), DEST_V1_1_0.getVersionId()), - SupportState.UNSUPPORTED); - verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V1_0_0.getVersionId(), DEST_V2_0_0.getVersionId()), + verify(mConfigRepository).listSourcesWithVersionIds(List.of(SRC_V0_1_0.getVersionId())); + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V0_1_0.getVersionId()), SupportState.UNSUPPORTED); + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(DEST_V0_1_0.getVersionId()), SupportState.DEPRECATED); + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V1_0_0.getVersionId(), DEST_V1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetUnsupportedVersionIdsAfterUpdate() { + final ActorDefinitionVersion ADV_0_1_0 = createActorDefinitionVersion(V0_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0).withSupportState(SupportState.DEPRECATED); + final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0).withSupportState(SupportState.DEPRECATED); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0).withSupportState(SupportState.SUPPORTED); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion(V3_0_0).withSupportState(SupportState.SUPPORTED); + + final List versionsBeforeUpdate = List.of( + ADV_0_1_0, + ADV_1_0_0, + ADV_1_1_0, + ADV_2_0_0, + ADV_3_0_0); + final SupportStateUpdate supportStateUpdate = new SupportStateUpdate( + List.of(ADV_1_0_0.getVersionId(), ADV_1_1_0.getVersionId()), + List.of(ADV_2_0_0.getVersionId()), + List.of()); + + final List unsupportedVersionIds = supportStateUpdater.getUnsupportedVersionIdsAfterUpdate(versionsBeforeUpdate, supportStateUpdate); + + final List expectedUnsupportedVersionIds = List.of( + ADV_0_1_0.getVersionId(), + ADV_1_0_0.getVersionId(), + ADV_1_1_0.getVersionId()); + + assertEquals(expectedUnsupportedVersionIds, unsupportedVersionIds); + } + + @Test + void testGetUnsupportedVersionIdsAfterUpdateWithRollback() { + final ActorDefinitionVersion ADV_0_1_0 = createActorDefinitionVersion(V0_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0).withSupportState(SupportState.DEPRECATED); + + final List versionsBeforeUpdate = List.of( + ADV_0_1_0, + ADV_1_0_0, + ADV_1_1_0, + ADV_2_0_0); + final SupportStateUpdate supportStateUpdate = new SupportStateUpdate( + List.of(), + List.of(ADV_1_0_0.getVersionId(), ADV_1_1_0.getVersionId()), + List.of(ADV_2_0_0.getVersionId())); + + final List unsupportedVersionIds = supportStateUpdater.getUnsupportedVersionIdsAfterUpdate(versionsBeforeUpdate, supportStateUpdate); + + final List expectedUnsupportedVersionIds = List.of(ADV_0_1_0.getVersionId()); + assertEquals(expectedUnsupportedVersionIds, unsupportedVersionIds); } @Test @@ -178,7 +280,7 @@ void testGetSupportStateUpdate() { final ActorDefinitionBreakingChange BC_1_0_0 = createBreakingChange(V1_0_0, "2021-02-01"); final ActorDefinitionBreakingChange BC_2_0_0 = createBreakingChange(V2_0_0, "2022-02-01"); - final ActorDefinitionBreakingChange BC_3_0_0 = createBreakingChange("3.0.0", "2024-01-01"); + final ActorDefinitionBreakingChange BC_3_0_0 = createBreakingChange(V3_0_0, "2024-01-01"); final ActorDefinitionBreakingChange BC_4_0_0 = createBreakingChange("4.0.0", "2025-01-01"); final ActorDefinitionBreakingChange BC_5_0_0 = createBreakingChange("5.0.0", "2026-01-01"); @@ -194,7 +296,7 @@ void testGetSupportStateUpdate() { final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0); final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); final ActorDefinitionVersion ADV_2_1_0 = createActorDefinitionVersion("2.1.0").withSupportState(SupportState.DEPRECATED); - final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion("3.0.0"); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion(V3_0_0); final ActorDefinitionVersion ADV_3_1_0 = createActorDefinitionVersion("3.1.0"); final ActorDefinitionVersion ADV_4_0_0 = createActorDefinitionVersion("4.0.0"); final ActorDefinitionVersion ADV_4_1_0 = createActorDefinitionVersion("4.1.0").withSupportState(SupportState.SUPPORTED); @@ -226,6 +328,7 @@ void testGetSupportStateUpdate() { assertEquals(expectedSupportStateUpdate, supportStateUpdate); verifyNoInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test @@ -254,6 +357,184 @@ void testGetSupportStateUpdateNoBreakingChanges() { assertEquals(expectedSupportStateUpdate, supportStateUpdate); verifyNoInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetSyncsToDisableForSource() throws JsonValidationException, ConfigNotFoundException, IOException { + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(ACTOR_DEFINITION_ID); + + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); + final List unsupportedVersionIds = List.of(ADV_1_0_0.getVersionId(), ADV_2_0_0.getVersionId()); + + final SourceConnection sourceConnection = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + final SourceConnection sourceConnection2 = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + final SourceConnection sourceConnection3 = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID_2) + .withSourceId(UUID.randomUUID()); + + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, false)); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection2.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID_2, sourceConnection3.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + + final SourceConnection sourceWithOverride = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceWithOverride.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, true)); + + when(mConfigRepository.listSourcesWithVersionIds(unsupportedVersionIds)) + .thenReturn(List.of(sourceConnection, sourceConnection2, sourceConnection3, sourceWithOverride)); + + final StandardSyncQuery workspaceQuery1 = new StandardSyncQuery( + WORKSPACE_ID, + List.of(sourceConnection.getSourceId(), sourceConnection2.getSourceId()), + null, false); + final List workspaceSyncs1 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery1)) + .thenReturn(workspaceSyncs1); + + final StandardSyncQuery workspaceQuery2 = new StandardSyncQuery( + WORKSPACE_ID_2, + List.of(sourceConnection3.getSourceId()), + null, false); + final List workspaceSyncs2 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery2)) + .thenReturn(workspaceSyncs2); + + final List expectedSyncIds = Stream.of( + workspaceSyncs1.get(0).getConnectionId(), + workspaceSyncs1.get(1).getConnectionId(), + workspaceSyncs1.get(2).getConnectionId(), + workspaceSyncs2.get(0).getConnectionId()).sorted().toList(); + + final List syncsToDisable = supportStateUpdater.getSyncsToDisableForSource(sourceDefinition, unsupportedVersionIds); + final List actualSyncIds = syncsToDisable.stream().map(StandardSync::getConnectionId).sorted().toList(); + assertEquals(expectedSyncIds, actualSyncIds); + + verify(mConfigRepository).listSourcesWithVersionIds(unsupportedVersionIds); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery1); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery2); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection2.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceWithOverride.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID_2, sourceConnection3.getSourceId()); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetSyncsToDisableForDestination() throws JsonValidationException, ConfigNotFoundException, IOException { + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withDestinationDefinitionId(ACTOR_DEFINITION_ID); + + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); + final List unsupportedVersionIds = List.of(ADV_1_0_0.getVersionId(), ADV_2_0_0.getVersionId()); + + final DestinationConnection destinationConnection = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + final DestinationConnection destinationConnection3 = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID_2) + .withDestinationId(UUID.randomUUID()); + + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, false)); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection2.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID_2, + destinationConnection3.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + + final DestinationConnection destinationWithOverride = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationWithOverride.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, true)); + + when(mConfigRepository.listDestinationsWithVersionIds(unsupportedVersionIds)) + .thenReturn(List.of(destinationConnection, destinationConnection2, destinationConnection3, destinationWithOverride)); + + final StandardSyncQuery workspaceQuery1 = new StandardSyncQuery( + WORKSPACE_ID, + null, + List.of(destinationConnection.getDestinationId(), destinationConnection2.getDestinationId()), + false); + final List workspaceSyncs1 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery1)) + .thenReturn(workspaceSyncs1); + + final StandardSyncQuery workspaceQuery2 = new StandardSyncQuery( + WORKSPACE_ID_2, + null, + List.of(destinationConnection3.getDestinationId()), + false); + final List workspaceSyncs2 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery2)) + .thenReturn(workspaceSyncs2); + + final List expectedSyncIds = Stream.of( + workspaceSyncs1.get(0).getConnectionId(), + workspaceSyncs1.get(1).getConnectionId(), + workspaceSyncs1.get(2).getConnectionId(), + workspaceSyncs2.get(0).getConnectionId()).sorted().toList(); + + final List syncsToDisable = supportStateUpdater.getSyncsToDisableForDestination(destinationDefinition, unsupportedVersionIds); + final List actualSyncIds = syncsToDisable.stream().map(StandardSync::getConnectionId).sorted().toList(); + assertEquals(expectedSyncIds, actualSyncIds); + + verify(mConfigRepository).listDestinationsWithVersionIds(unsupportedVersionIds); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery1); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery2); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection2.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationWithOverride.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID_2, + destinationConnection3.getDestinationId()); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testDisableSyncs() throws IOException { + final List syncsToDisable = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.ACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.ACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.INACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.DEPRECATED)); + + supportStateUpdater.disableSyncs(syncsToDisable); + + verify(mConfigRepository).writeStandardSync(syncsToDisable.get(0).withStatus(Status.INACTIVE)); + verify(mConfigRepository).writeStandardSync(syncsToDisable.get(1).withStatus(Status.INACTIVE)); + verifyNoMoreInteractions(mConfigRepository); } } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java index 9f4413811ac..82ac86fbbf0 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java @@ -6,7 +6,6 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; -import io.airbyte.config.User.AuthProvider; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.Optional; @@ -48,7 +47,7 @@ void getUserByIdTest() throws IOException { @Test void getUserByAuthIdTest() throws IOException { for (final User user : MockData.users()) { - final Optional userFromDb = userPersistence.getUserByAuthId(user.getAuthUserId(), AuthProvider.GOOGLE_IDENTITY_PLATFORM); + final Optional userFromDb = userPersistence.getUserByAuthId(user.getAuthUserId(), user.getAuthProvider()); Assertions.assertEquals(user, userFromDb.get()); } } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java index 81b88b67527..0c1db316d07 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java @@ -6,6 +6,7 @@ import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE; import static io.airbyte.db.instance.jobs.jooq.generated.Tables.JOBS; @@ -13,6 +14,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType; import io.airbyte.db.instance.configs.jooq.generated.enums.NamespaceDefinitionType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import java.io.IOException; import java.sql.SQLException; import java.time.OffsetDateTime; @@ -29,6 +31,8 @@ class WorkspaceFilterTest extends BaseConfigDatabaseTest { private static final UUID SRC_DEF_ID = UUID.randomUUID(); private static final UUID DST_DEF_ID = UUID.randomUUID(); + private static final UUID SRC_DEF_VER_ID = UUID.randomUUID(); + private static final UUID DST_DEF_VER_ID = UUID.randomUUID(); private static final UUID ACTOR_ID_0 = UUID.randomUUID(); private static final UUID ACTOR_ID_1 = UUID.randomUUID(); private static final UUID ACTOR_ID_2 = UUID.randomUUID(); @@ -53,21 +57,31 @@ static void setUpAll() throws SQLException { .values(DST_DEF_ID, "dstDef", ActorType.destination) .values(UUID.randomUUID(), "dstDef", ActorType.destination) .execute()); + // create actor_definition_version + database.transaction(ctx -> ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC, + ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .execute()); // create workspace - database.transaction(ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.SLUG, WORKSPACE.INITIAL_SETUP_COMPLETE) - .values(WORKSPACE_ID_0, "ws-0", "ws-0", true) - .values(WORKSPACE_ID_1, "ws-1", "ws-1", true) - .values(WORKSPACE_ID_2, "ws-2", "ws-2", true) - .values(WORKSPACE_ID_3, "ws-3", "ws-3", true) - .execute()); + database.transaction( + ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.SLUG, WORKSPACE.INITIAL_SETUP_COMPLETE, WORKSPACE.TOMBSTONE) + .values(WORKSPACE_ID_0, "ws-0", "ws-0", true, false) + .values(WORKSPACE_ID_1, "ws-1", "ws-1", true, false) + .values(WORKSPACE_ID_2, "ws-2", "ws-2", true, true) // note that workspace 2 is tombstoned! + .values(WORKSPACE_ID_3, "ws-3", "ws-3", true, true) // note that workspace 3 is tombstoned! + .execute()); // create actors database.transaction( - ctx -> ctx.insertInto(ACTOR, ACTOR.WORKSPACE_ID, ACTOR.ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE) - .values(WORKSPACE_ID_0, ACTOR_ID_0, SRC_DEF_ID, "ACTOR-0", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_1, ACTOR_ID_1, SRC_DEF_ID, "ACTOR-1", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_2, ACTOR_ID_2, DST_DEF_ID, "ACTOR-2", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_3, ACTOR_ID_3, DST_DEF_ID, "ACTOR-3", JSONB.valueOf("{}"), ActorType.source) + ctx -> ctx + .insertInto(ACTOR, ACTOR.WORKSPACE_ID, ACTOR.ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE) + .values(WORKSPACE_ID_0, ACTOR_ID_0, SRC_DEF_ID, SRC_DEF_VER_ID, "ACTOR-0", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_1, ACTOR_ID_1, SRC_DEF_ID, SRC_DEF_VER_ID, "ACTOR-1", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_2, ACTOR_ID_2, DST_DEF_ID, DST_DEF_VER_ID, "ACTOR-2", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_3, ACTOR_ID_3, DST_DEF_ID, DST_DEF_VER_ID, "ACTOR-3", JSONB.valueOf("{}"), ActorType.source) .execute()); // create connections database.transaction( @@ -101,8 +115,8 @@ void beforeEach() { } @Test - @DisplayName("Should return a list of workspace IDs with most recently running jobs") - void testListWorkspacesByMostRecentlyRunningJobs() throws IOException { + @DisplayName("Should return a list of active workspace IDs with most recently running jobs") + void testListActiveWorkspacesByMostRecentlyRunningJobs() throws IOException { final int timeWindowInHours = 48; /* * Following function is to filter workspace (IDs) with most recently running jobs within a given @@ -110,17 +124,16 @@ void testListWorkspacesByMostRecentlyRunningJobs() throws IOException { * time window. Step 2: Trace back via CONNECTION table and ACTOR table. Step 3: Return workspace * IDs from ACTOR table. */ - final List actualResult = configRepository.listWorkspacesByMostRecentlyRunningJobs(timeWindowInHours); + final List actualResult = configRepository.listActiveWorkspacesByMostRecentlyRunningJobs(timeWindowInHours); /* * With the test data provided above, expected outputs for each step: Step 1: `jobs` (IDs) OL, 1L, * 2L, 3L, 4L, 5L and 6L. Step 2: `connections` (IDs) CONN_ID_0, CONN_ID_1, CONN_ID_2, CONN_ID_3, * and CONN_ID_4 `actors` (IDs) ACTOR_ID_0, ACTOR_ID_1, and ACTOR_ID_2. Step 3: `workspaces` (IDs) - * WORKSPACE_ID_0, WORKSPACE_ID_1 and WORKSPACE_ID_2. + * WORKSPACE_ID_0, WORKSPACE_ID_1. Note that WORKSPACE_ID_2 is excluded because it is tombstoned. */ final List expectedResult = new ArrayList<>(); expectedResult.add(WORKSPACE_ID_0); expectedResult.add(WORKSPACE_ID_1); - expectedResult.add(WORKSPACE_ID_2); assertTrue(expectedResult.size() == actualResult.size() && expectedResult.containsAll(actualResult) && actualResult.containsAll(expectedResult)); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java index 2281164ffc8..57394a517b5 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java @@ -18,13 +18,19 @@ import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; import io.airbyte.config.Organization; +import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.ReleaseStage; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; +import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; @@ -49,11 +55,15 @@ class WorkspacePersistenceTest extends BaseConfigDatabaseTest { private ConfigRepository configRepository; private WorkspacePersistence workspacePersistence; + private PermissionPersistence permissionPersistence; + private UserPersistence userPersistence; @BeforeEach void setup() throws Exception { configRepository = spy(new ConfigRepository(database, null, MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); workspacePersistence = new WorkspacePersistence(database); + permissionPersistence = new PermissionPersistence(database); + userPersistence = new UserPersistence(database); final OrganizationPersistence organizationPersistence = new OrganizationPersistence(database); truncateAllTables(); @@ -125,6 +135,7 @@ private static ActorDefinitionVersion createActorDefinitionVersion(final UUID ac return new ActorDefinitionVersion() .withActorDefinitionId(actorDefinitionId) .withDockerRepository("dockerhub") + .withSupportLevel(SupportLevel.COMMUNITY) .withDockerImageTag("0.0.1") .withReleaseStage(releaseStage); } @@ -203,10 +214,10 @@ void testWorkspaceHasAlphaOrBetaConnector(final ReleaseStage sourceReleaseStage, final StandardWorkspace workspace = createBaseStandardWorkspace(); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion( + configRepository.writeConnectorMetadata( createSourceDefinition(), createActorDefinitionVersion(SOURCE_DEFINITION_ID, sourceReleaseStage)); - configRepository.writeDestinationDefinitionAndDefaultVersion( + configRepository.writeConnectorMetadata( createDestinationDefinition(), createActorDefinitionVersion(DESTINATION_DEFINITION_ID, destinationReleaseStage)); @@ -227,7 +238,7 @@ void testListWorkspacesInOrgNoKeyword() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.empty()); assertReturnsWorkspace(createBaseStandardWorkspace().withTombstone(false)); @@ -247,7 +258,7 @@ void testListWorkspacesInOrgWithPagination() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 1, 0), Optional.empty()); assertEquals(1, workspaces.size()); @@ -266,11 +277,122 @@ void testListWorkspacesInOrgWithKeyword() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.of("keyword")); assertEquals(1, workspaces.size()); assertEquals(workspace, workspaces.get(0)); } + @Test + void testGetDefaultWorkspaceForOrganization() throws JsonValidationException, IOException { + final StandardWorkspace expectedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspaceInOrganization1"); + + configRepository.writeStandardWorkspaceNoSecrets(expectedWorkspace); + + final StandardWorkspace tombstonedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("tombstonedWorkspace") + .withTombstone(true); + + configRepository.writeStandardWorkspaceNoSecrets(tombstonedWorkspace); + + final StandardWorkspace laterWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("laterWorkspace"); + + configRepository.writeStandardWorkspaceNoSecrets(laterWorkspace); + + final StandardWorkspace actualWorkspace = workspacePersistence.getDefaultWorkspaceForOrganization(MockData.ORGANIZATION_ID_1); + + assertEquals(expectedWorkspace, actualWorkspace); + } + + @Test + void testListWorkspacesByUserIdWithKeywordWithPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1, name contains search "keyword" + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace_with_keyword_1"); + configRepository.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2, name contains search "Keyword" + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace_with_Keyword_2"); + configRepository.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserIdPaginated( + new ResourcesByUserQueryPaginated(userId, false, 10, 0), Optional.of("keyWord")); + + assertEquals(2, workspaces.size()); + } + + @Test + void testListWorkspacesByUserIdWithoutKeywordWithoutPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1 + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace1"); + configRepository.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2 + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace2"); + configRepository.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserId( + userId, false, Optional.empty()); + + assertEquals(2, workspaces.size()); + } + } diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java index f0f870960dd..f02d134d827 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java @@ -17,9 +17,10 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.IngestBreakingChanges; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.validation.json.JsonValidationException; @@ -27,7 +28,6 @@ import jakarta.inject.Named; import jakarta.inject.Singleton; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -51,18 +51,20 @@ public class ApplyDefinitionsHelper { private final JobPersistence jobPersistence; private final ConfigRepository configRepository; private final FeatureFlagClient featureFlagClient; + private final SupportStateUpdater supportStateUpdater; private int newConnectorCount; private int changedConnectorCount; - private List allBreakingChanges; private static final Logger LOGGER = LoggerFactory.getLogger(ApplyDefinitionsHelper.class); public ApplyDefinitionsHelper(@Named("seedDefinitionsProvider") final DefinitionsProvider definitionsProvider, final JobPersistence jobPersistence, final ConfigRepository configRepository, - final FeatureFlagClient featureFlagClient) { + final FeatureFlagClient featureFlagClient, + final SupportStateUpdater supportStateUpdater) { this.definitionsProvider = definitionsProvider; this.jobPersistence = jobPersistence; this.configRepository = configRepository; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -92,7 +94,6 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc newConnectorCount = 0; changedConnectorCount = 0; - allBreakingChanges = new ArrayList<>(); for (final ConnectorRegistrySourceDefinition def : protocolCompatibleSourceDefinitions) { applySourceDefinition(actorDefinitionIdsToDefaultVersionsMap, def, actorDefinitionIdsInUse, updateAll); @@ -100,8 +101,8 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc for (final ConnectorRegistryDestinationDefinition def : protocolCompatibleDestinationDefinitions) { applyDestinationDefinition(actorDefinitionIdsToDefaultVersionsMap, def, actorDefinitionIdsInUse, updateAll); } - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(allBreakingChanges); + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + supportStateUpdater.updateSupportStates(); } LOGGER.info("New connectors added: {}", newConnectorCount); @@ -127,13 +128,11 @@ private void applySourceDefinition(final Map actor return; } - allBreakingChanges.addAll(breakingChangesForDef); - final boolean connectorIsNew = !actorDefinitionIdsAndDefaultVersions.containsKey(newSourceDef.getSourceDefinitionId()); if (connectorIsNew) { LOGGER.info("Adding new connector {}:{}", newDef.getDockerRepository(), newDef.getDockerImageTag()); newConnectorCount++; - configRepository.writeSourceDefinitionAndDefaultVersion(newSourceDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newSourceDef, newADV, breakingChangesForDef); return; } @@ -146,7 +145,7 @@ private void applySourceDefinition(final Map actor currentDefaultADV.getDockerImageTag(), newADV.getDockerImageTag()); changedConnectorCount++; - configRepository.writeSourceDefinitionAndDefaultVersion(newSourceDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newSourceDef, newADV, breakingChangesForDef); } else { configRepository.updateStandardSourceDefinition(newSourceDef); } @@ -171,13 +170,11 @@ private void applyDestinationDefinition(final Map return; } - allBreakingChanges.addAll(breakingChangesForDef); - final boolean connectorIsNew = !actorDefinitionIdsAndDefaultVersions.containsKey(newDestinationDef.getDestinationDefinitionId()); if (connectorIsNew) { LOGGER.info("Adding new connector {}:{}", newDef.getDockerRepository(), newDef.getDockerImageTag()); newConnectorCount++; - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestinationDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestinationDef, newADV, breakingChangesForDef); return; } @@ -190,7 +187,7 @@ private void applyDestinationDefinition(final Map currentDefaultADV.getDockerImageTag(), newADV.getDockerImageTag()); changedConnectorCount++; - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestinationDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestinationDef, newADV, breakingChangesForDef); } else { configRepository.updateStandardDestinationDefinition(newDestinationDef); } diff --git a/airbyte-config/init/src/main/resources/icons/chroma.svg b/airbyte-config/init/src/main/resources/icons/chroma.svg new file mode 100644 index 00000000000..fc07764e435 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/chroma.svg @@ -0,0 +1,59 @@ + + + + + Chroma + + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/milvus.svg b/airbyte-config/init/src/main/resources/icons/milvus.svg new file mode 100644 index 00000000000..b4e13796df2 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/milvus.svg @@ -0,0 +1,60 @@ + + + + + + + milvus-icon-color + + + + + + milvus-icon-color + + + + diff --git a/airbyte-config/init/src/main/resources/icons/qdrant.svg b/airbyte-config/init/src/main/resources/icons/qdrant.svg new file mode 100644 index 00000000000..ce295923d9d --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/qdrant.svg @@ -0,0 +1,97 @@ + + + + qdrant + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/icons/weaviate.svg b/airbyte-config/init/src/main/resources/icons/weaviate.svg new file mode 100644 index 00000000000..ff31522d53b --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/weaviate.svg @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java index d13b8cd68dd..8b9b275b90e 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java +++ b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java @@ -5,7 +5,6 @@ package io.airbyte.config.init; import static io.airbyte.featureflag.ContextKt.ANONYMOUS; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -14,7 +13,6 @@ import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.Version; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.BreakingChanges; import io.airbyte.config.ConnectorRegistryDestinationDefinition; @@ -24,16 +22,16 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.IngestBreakingChanges; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -54,6 +52,7 @@ class ApplyDefinitionsHelperTest { private ConfigRepository configRepository; private DefinitionsProvider definitionsProvider; private JobPersistence jobPersistence; + private SupportStateUpdater supportStateUpdater; private FeatureFlagClient featureFlagClient; private ApplyDefinitionsHelper applyDefinitionsHelper; @@ -101,11 +100,13 @@ void setup() { configRepository = mock(ConfigRepository.class); definitionsProvider = mock(DefinitionsProvider.class); jobPersistence = mock(JobPersistence.class); + supportStateUpdater = mock(SupportStateUpdater.class); featureFlagClient = mock(TestClient.class); - applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobPersistence, configRepository, featureFlagClient); + applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobPersistence, configRepository, featureFlagClient, supportStateUpdater); - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); // Default calls to empty. when(definitionsProvider.getDestinationDefinitions()).thenReturn(Collections.emptyList()); @@ -134,17 +135,17 @@ void testNewConnectorIsWritten(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3)); - verify(configRepository).writeActorDefinitionBreakingChanges(List.of()); + verify(supportStateUpdater).updateSupportStates(); - verifyNoMoreInteractions(configRepository); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -161,17 +162,17 @@ void testConnectorIsUpdatedIfItIsNotInUse(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); + verify(supportStateUpdater).updateSupportStates(); - verifyNoMoreInteractions(configRepository); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -188,21 +189,21 @@ void testUpdateBehaviorIfConnectorIsInUse(final boolean updateAll) verifyConfigRepositoryGetInteractions(); if (updateAll) { - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); } else { verify(configRepository).updateStandardSourceDefinition(ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2)); verify(configRepository).updateStandardDestinationDefinition(ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); } - verifyNoMoreInteractions(configRepository); + verify(supportStateUpdater).updateSupportStates(); + + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -222,30 +223,31 @@ void testDefinitionsFiltering(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository, never()).writeSourceDefinitionAndDefaultVersion( + verify(configRepository, never()).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(postgresWithOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionVersion(s3withOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(postgresWithOldProtocolVersion)); - verify(configRepository, never()).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository, never()).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(s3withOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionVersion(postgresWithOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(s3withOldProtocolVersion)); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); - verifyNoMoreInteractions(configRepository); + verify(supportStateUpdater).updateSupportStates(); + + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @Test - void testTurnOffIngestBreakingChangesFeatureFlag() throws JsonValidationException, IOException, ConfigNotFoundException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(false); + void testTurnOffRunSupportStateUpdaterFeatureFlag() throws JsonValidationException, ConfigNotFoundException, IOException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(false); when(definitionsProvider.getSourceDefinitions()).thenReturn(List.of(SOURCE_POSTGRES_2)); when(definitionsProvider.getDestinationDefinitions()).thenReturn(List.of(DESTINATION_S3_2)); @@ -253,24 +255,17 @@ void testTurnOffIngestBreakingChangesFeatureFlag() throws JsonValidationExceptio applyDefinitionsHelper.apply(true); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository, never()).writeActorDefinitionBreakingChanges(any()); - - verifyNoMoreInteractions(configRepository); - } - private static List getExpectedBreakingChanges() { - final List breakingChanges = new ArrayList<>(); - breakingChanges.addAll(ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - breakingChanges.addAll(ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - return breakingChanges; + verify(supportStateUpdater, never()).updateSupportStates(); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } } diff --git a/airbyte-config/specs/build.gradle b/airbyte-config/specs/build.gradle index a64e351fb8b..ee2018e2721 100644 --- a/airbyte-config/specs/build.gradle +++ b/airbyte-config/specs/build.gradle @@ -20,6 +20,7 @@ dependencies { implementation project(':airbyte-config:config-models') implementation libs.airbyte.protocol implementation project(':airbyte-json-validation') + implementation libs.okhttp testRuntimeOnly libs.junit.jupiter.engine testImplementation libs.bundles.junit diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java index 9d9002a9fa1..7f8444984d8 100644 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java +++ b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java @@ -4,8 +4,9 @@ package io.airbyte.config.specs; +import static io.micronaut.http.HttpHeaders.ACCEPT; + import com.fasterxml.jackson.databind.JsonNode; -import com.google.api.client.http.HttpStatusCodes; import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.constants.AirbyteCatalogConstants; import io.airbyte.commons.json.Jsons; @@ -18,13 +19,13 @@ import io.micronaut.cache.annotation.CacheConfig; import io.micronaut.cache.annotation.Cacheable; import io.micronaut.context.annotation.Value; +import io.micronaut.http.MediaType; import jakarta.inject.Singleton; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; +import java.net.URL; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -32,6 +33,9 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,15 +48,15 @@ public class RemoteDefinitionsProvider implements DefinitionsProvider { private static final Logger LOGGER = LoggerFactory.getLogger(RemoteDefinitionsProvider.class); - private static final HttpClient httpClient = HttpClient.newHttpClient(); + private final OkHttpClient okHttpClient; + private final URI remoteRegistryBaseUrl; private final DeploymentMode deploymentMode; - private final Duration timeout; + private static final int NOT_FOUND = 404; private URI parsedRemoteRegistryBaseUrlOrDefault(final String remoteRegistryBaseUrl) { try { if (remoteRegistryBaseUrl == null || remoteRegistryBaseUrl.isEmpty()) { - LOGGER.error("Remote connector registry base URL cannot be null or empty. Falling back to default"); return new URI(AirbyteCatalogConstants.REMOTE_REGISTRY_BASE_URL); } else { return new URI(remoteRegistryBaseUrl); @@ -70,8 +74,10 @@ public RemoteDefinitionsProvider(@Value("${airbyte.connector-registry.remote.bas LOGGER.info("Creating remote definitions provider for URL '{}' and registry '{}'...", remoteRegistryBaseUrlUri, deploymentMode); this.remoteRegistryBaseUrl = remoteRegistryBaseUrlUri; - this.timeout = Duration.ofMillis(remoteCatalogTimeoutMs); this.deploymentMode = deploymentMode; + this.okHttpClient = new OkHttpClient.Builder() + .callTimeout(Duration.ofMillis(remoteCatalogTimeoutMs)) + .build(); } private Map getSourceDefinitionsMap() { @@ -144,13 +150,21 @@ private String getRegistryName() { } @VisibleForTesting - URI getRegistryUrl() { - return remoteRegistryBaseUrl.resolve(String.format("registries/v0/%s_registry.json", getRegistryName())); + URL getRegistryUrl() { + try { + return new URL(remoteRegistryBaseUrl + String.format("registries/v0/%s_registry.json", getRegistryName())); + } catch (final MalformedURLException e) { + throw new RuntimeException("Invalid URL format", e); + } } @VisibleForTesting - URI getRegistryEntryUrl(final String connectorName, final String version) { - return remoteRegistryBaseUrl.resolve(String.format("metadata/%s/%s/%s.json", connectorName, version, getRegistryName())); + URL getRegistryEntryUrl(final String connectorName, final String version) { + try { + return remoteRegistryBaseUrl.resolve(String.format("metadata/%s/%s/%s.json", connectorName, version, getRegistryName())).toURL(); + } catch (final MalformedURLException e) { + throw new RuntimeException("Invalid URL format", e); + } } /** @@ -160,17 +174,20 @@ URI getRegistryEntryUrl(final String connectorName, final String version) { */ @Cacheable public ConnectorRegistry getRemoteConnectorRegistry() { - try { - final HttpRequest request = HttpRequest.newBuilder(getRegistryUrl()).timeout(timeout).header("accept", "application/json").build(); - - final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (errorStatusCode(response)) { + final Request request = new Request.Builder() + .url(getRegistryUrl()) + .header(ACCEPT, MediaType.APPLICATION_JSON) + .build(); + + try (Response response = okHttpClient.newCall(request).execute()) { + if (response.isSuccessful() && response.body() != null) { + final String responseBody = response.body().string(); + LOGGER.info("Fetched latest remote definitions ({})", responseBody.hashCode()); + return Jsons.deserialize(responseBody, ConnectorRegistry.class); + } else { throw new IOException( - "getRemoteConnectorRegistry request ran into status code error: " + response.statusCode() + " with message: " + response.getClass()); + "getRemoteConnectorRegistry request ran into status code error: " + response.code() + " with message: " + response.message()); } - - LOGGER.info("Fetched latest remote definitions ({})", response.body().hashCode()); - return Jsons.deserialize(response.body(), ConnectorRegistry.class); } catch (final Exception e) { throw new RuntimeException("Failed to fetch remote connector registry", e); } @@ -178,27 +195,24 @@ public ConnectorRegistry getRemoteConnectorRegistry() { @VisibleForTesting Optional getConnectorRegistryEntryJson(final String connectorName, final String version) { - try { - final URI registryEntryPath = getRegistryEntryUrl(connectorName, version); - final HttpRequest request = HttpRequest.newBuilder(registryEntryPath).timeout(timeout).header("accept", "application/json").build(); - - final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { - // 404s mean we don't have a registry entry for this connector version + final URL registryEntryPath = getRegistryEntryUrl(connectorName, version); + final Request request = new Request.Builder() + .url(registryEntryPath) + .header(ACCEPT, MediaType.APPLICATION_JSON) + .build(); + + try (Response response = okHttpClient.newCall(request).execute()) { + if (response.code() == NOT_FOUND) { return Optional.empty(); - } else if (errorStatusCode(response)) { + } else if (response.isSuccessful() && response.body() != null) { + return Optional.of(Jsons.deserialize(response.body().string())); + } else { throw new IOException( - "getConnectorRegistryEntry request ran into status code error: " + response.statusCode() + " with message: " + response.getClass()); + "getConnectorRegistryEntry request ran into status code error: " + response.code() + " with message: " + response.message()); } - - return Optional.of(Jsons.deserialize(response.body())); - } catch (final IOException | InterruptedException e) { + } catch (final IOException e) { throw new RuntimeException(String.format("Failed to fetch connector registry entry for %s:%s", connectorName, version), e); } } - private static Boolean errorStatusCode(final HttpResponse response) { - return response.statusCode() >= 400; - } - } diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java index cac20d55d43..e8b9b32c178 100644 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java +++ b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java @@ -15,11 +15,12 @@ import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.ConnectorRegistryDestinationDefinition; import io.airbyte.config.ConnectorRegistrySourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.URI; import java.net.URL; -import java.net.http.HttpTimeoutException; import java.nio.charset.Charset; import java.util.List; import java.util.Optional; @@ -79,6 +80,7 @@ void testGetSourceDefinition() throws Exception { assertEquals(URI.create("https://docs.airbyte.io/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); assertEquals(false, stripeSource.getTombstone()); assertEquals("0.2.1", stripeSource.getProtocolVersion()); + assertEquals(SupportLevel.COMMUNITY, stripeSource.getSupportLevel()); } @Test @@ -97,10 +99,11 @@ void testGetDestinationDefinition() throws Exception { assertEquals(URI.create("https://docs.airbyte.io/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); assertEquals(false, s3Destination.getTombstone()); assertEquals("0.2.2", s3Destination.getProtocolVersion()); + assertEquals(SupportLevel.COMMUNITY, s3Destination.getSupportLevel()); } @Test - void testGetInvalidDefinitionId() throws Exception { + void testGetInvalidDefinitionId() { webServer.enqueue(validCatalogResponse); webServer.enqueue(validCatalogResponse); @@ -117,7 +120,7 @@ void testGetInvalidDefinitionId() throws Exception { } @Test - void testGetSourceDefinitions() throws Exception { + void testGetSourceDefinitions() { webServer.enqueue(validCatalogResponse); final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(baseUrl, DEPLOYMENT_MODE, TimeUnit.SECONDS.toMillis(30)); @@ -128,7 +131,7 @@ void testGetSourceDefinitions() throws Exception { } @Test - void testGetDestinationDefinitions() throws Exception { + void testGetDestinationDefinitions() { webServer.enqueue(validCatalogResponse); final RemoteDefinitionsProvider remoteDefinitionsProvider = new RemoteDefinitionsProvider(baseUrl, DEPLOYMENT_MODE, TimeUnit.SECONDS.toMillis(30)); @@ -157,7 +160,7 @@ void testTimeOut() { }); assertTrue(ex.getMessage().contains("Failed to fetch remote connector registry")); - assertTrue(ex.getCause() instanceof HttpTimeoutException); + assertEquals(ex.getCause().getClass(), InterruptedIOException.class); } @Test @@ -175,7 +178,7 @@ void testGetRegistryUrl(final String deploymentMode) { final String baseUrl = "https://connectors.airbyte.com/files/"; final RemoteDefinitionsProvider definitionsProvider = new RemoteDefinitionsProvider(baseUrl, DeploymentMode.valueOf(deploymentMode), TimeUnit.SECONDS.toMillis(1)); - final URI registryUrl = definitionsProvider.getRegistryUrl(); + final URL registryUrl = definitionsProvider.getRegistryUrl(); assertEquals(String.format("https://connectors.airbyte.com/files/registries/v0/%s_registry.json", deploymentMode.toLowerCase()), registryUrl.toString()); } @@ -188,7 +191,7 @@ void testGetRegistryEntryUrl(final String deploymentMode) { final String version = "1.0.0"; final RemoteDefinitionsProvider definitionsProvider = new RemoteDefinitionsProvider(baseUrl, DeploymentMode.valueOf(deploymentMode), TimeUnit.SECONDS.toMillis(1)); - final URI registryEntryUrl = definitionsProvider.getRegistryEntryUrl(connectorName, version); + final URL registryEntryUrl = definitionsProvider.getRegistryEntryUrl(connectorName, version); assertEquals(String.format("https://connectors.airbyte.com/files/metadata/airbyte/source-github/1.0.0/%s.json", deploymentMode.toLowerCase()), registryEntryUrl.toString()); } diff --git a/airbyte-config/specs/src/test/resources/connector_catalog.json b/airbyte-config/specs/src/test/resources/connector_catalog.json index f4e09b0d95c..1fd287a9779 100644 --- a/airbyte-config/specs/src/test/resources/connector_catalog.json +++ b/airbyte-config/specs/src/test/resources/connector_catalog.json @@ -104,7 +104,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "b4c5d105-31fd-4817-96b6-cb923bfc04cb", @@ -207,6 +208,7 @@ "public": true, "custom": false, "releaseStage": "alpha", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -425,6 +427,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -634,6 +637,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -720,7 +724,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "81740ce8-d764-4ea7-94df-16bb41de36ae", @@ -766,7 +771,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ce0d828e-1dc4-496c-b122-2da42e637e48", @@ -952,7 +958,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "072d5540-f236-4294-ba7c-ade8fd918496", @@ -1141,7 +1148,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "8ccd8909-4e99-4141-b48d-4984b70b2d89", @@ -1234,7 +1242,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "68f351a7-2745-4bef-ad7f-996b8e51bb8c", @@ -1338,7 +1347,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3", @@ -1373,7 +1383,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "18081484-02a5-4662-8dba-b270b582f321", @@ -1492,7 +1503,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ca8f6566-e555-4b40-943a-545bf123117a", @@ -1891,6 +1903,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -1939,7 +1952,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "356668e2-7e34-47f3-a3b0-67a8a481b692", @@ -1982,7 +1996,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a", @@ -2055,7 +2070,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "d4353156-9217-4cad-8dd7-c108fd4f74cf", @@ -2294,7 +2310,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "294a4790-429b-40ae-9516-49826b9702e1", @@ -2470,7 +2487,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", @@ -2510,7 +2528,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c", @@ -2666,7 +2685,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ca81ee7c-3163-4246-af40-094cc31e5e42", @@ -2848,7 +2868,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", @@ -3038,7 +3059,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", @@ -3351,7 +3373,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "2340cbba-358e-11ec-8d3d-0242ac130203", @@ -3499,7 +3522,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "e06ad785-ad6f-4647-b2e8-3027a5c59454", @@ -3559,7 +3583,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", @@ -3623,7 +3648,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", @@ -3851,6 +3877,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -3912,7 +3939,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", @@ -4315,6 +4343,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -4388,7 +4417,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "3dc6f384-cd6b-4be3-ad16-a41450899bf0", @@ -4456,7 +4486,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", @@ -4958,6 +4989,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -5027,7 +5059,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212", @@ -5076,7 +5109,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", @@ -5211,7 +5245,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "983fd355-6bf3-4709-91b5-37afa391eeb6", @@ -5330,7 +5365,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460", @@ -5581,7 +5617,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "fa9f58c6-2d03-4237-aaa4-07d75e0c1396", @@ -5627,7 +5664,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "47f17145-fe20-4ef5-a548-e29b048adf84", @@ -5664,7 +5702,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "798ae795-5189-42b6-b64e-3cb91db93338", @@ -5714,7 +5753,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "59c5501b-9f95-411e-9269-7143c939adbd", @@ -5759,7 +5799,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c", @@ -5803,7 +5844,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "47f25999-dd5e-4636-8c39-e7cea2453331", @@ -5937,7 +5979,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "63cea06f-1c75-458d-88fe-ad48c7cb27fd", @@ -6002,7 +6045,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "686473f1-76d9-4994-9cc7-9b13da46147c", @@ -6055,7 +6099,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "b6604cbd-1b12-4c08-8767-e140d0fb0877", @@ -6102,7 +6147,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "bad83517-5e54-4a3d-9b53-63e85fbd4d7c", @@ -6273,7 +6319,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "dfffecb7-9a13-43e9-acdc-b92af7997ca9", @@ -6312,7 +6359,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635", @@ -6354,7 +6402,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7", @@ -6399,7 +6448,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "72d405a3-56d8-499f-a571-667c03406e43", @@ -6432,7 +6482,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd", @@ -6550,7 +6601,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", @@ -6598,7 +6650,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", @@ -6957,7 +7010,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "dfd88b22-b603-4c3d-aad7-3701784586b1", @@ -7014,7 +7068,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", @@ -7261,7 +7316,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "6f2ac653-8623-43c4-8950-19218c7caf3d", @@ -7319,7 +7375,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", @@ -7372,7 +7429,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", @@ -7523,7 +7581,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80", @@ -7577,7 +7636,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50", @@ -7730,7 +7790,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", @@ -7861,7 +7922,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246", @@ -7897,7 +7959,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "eb4c9e00-db83-4d63-a386-39cfa91012a8", @@ -8031,7 +8094,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7", @@ -8141,7 +8205,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734", @@ -8183,7 +8248,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", @@ -8215,7 +8281,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", @@ -8343,7 +8410,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "6acf6b55-4f1e-4fca-944e-1a3caef8aba8", @@ -8392,7 +8460,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", @@ -8440,7 +8509,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "2e875208-0c0b-4ee4-9e92-1cb3156ea799", @@ -8480,7 +8550,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "95e8cffd-b8c4-4039-968e-d32fb4a69bde", @@ -8520,7 +8591,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3", @@ -8558,7 +8630,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "789f8e7a-2d28-11ec-8d3d-0242ac130003", @@ -8589,7 +8662,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", @@ -8692,7 +8766,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e", @@ -8745,7 +8820,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", @@ -8801,7 +8877,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002", @@ -8925,7 +9002,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", @@ -8987,7 +9065,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "c7cb421b-942e-4468-99ee-e369bcabaec5", @@ -9036,7 +9115,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", @@ -9314,7 +9394,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227", @@ -9480,7 +9561,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", @@ -9566,7 +9648,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "80a54ea2-9959-4040-aac1-eee42423ec9b", @@ -9712,7 +9795,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e", @@ -9844,7 +9928,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "722ba4bf-06ec-45a4-8dd5-72e4a5cf3903", @@ -9899,7 +9984,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", @@ -10094,7 +10180,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "1d4fdb25-64fc-4569-92da-fcdca79a8372", @@ -10246,7 +10333,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", @@ -10524,7 +10612,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "7f0455fb-4518-4ec0-b7a3-d808bf8081cc", @@ -10592,7 +10681,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a", @@ -10639,7 +10729,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", @@ -10697,7 +10788,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "3052c77e-8b91-47e2-97a0-a29a22794b4b", @@ -10729,7 +10821,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", @@ -10763,7 +10856,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f", @@ -10809,7 +10903,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", @@ -11201,7 +11296,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d60a46d4-709f-4092-a6b7-2457f7d455f5", @@ -11237,7 +11333,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b08e4776-d1de-4e80-ab5c-1e51dad934a2", @@ -11300,7 +11397,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", @@ -11340,7 +11438,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01", @@ -11388,7 +11487,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", @@ -11462,7 +11562,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "db04ecd1-42e7-4115-9cec-95812905c626", @@ -11591,7 +11692,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2", @@ -11888,7 +11990,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "a827c52e-791c-4135-a245-e233c5255199", @@ -12008,7 +12111,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b117307c-14b6-41aa-9422-947e34922962", @@ -12160,7 +12264,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77", @@ -12234,7 +12339,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", @@ -12274,7 +12380,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "cdaf146a-9b75-49fd-9dd2-9d64a0bb4781", @@ -12323,7 +12430,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", @@ -12475,7 +12583,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "2fed2292-5586-480c-af92-9944e39fe12d", @@ -12519,7 +12628,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "374ebc65-6636-4ea0-925c-7d35999a8ffc", @@ -12603,7 +12713,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b", @@ -12675,7 +12786,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", @@ -12876,7 +12988,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", @@ -13039,7 +13152,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", @@ -13104,7 +13218,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5", @@ -13164,7 +13279,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "d1aa448b-7c54-498e-ad95-263cbebcd2db", @@ -13197,7 +13313,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35", @@ -13366,7 +13483,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "b9dc6155-672e-42ea-b10d-9f1f1fb95ab1", @@ -13424,7 +13542,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e7eff203-90bf-43e5-a240-19ea3056c474", @@ -13471,7 +13590,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "c4cfaeda-c757-489a-8aba-859fb08b6970", @@ -13522,7 +13642,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "ef580275-d9a9-48bb-af5e-db0f5855be04", @@ -13564,7 +13685,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "afa734e4-3571-11ec-991a-1e0031268139", @@ -13625,7 +13747,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4", @@ -13778,7 +13901,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "325e0640-e7b3-4e24-b823-3361008f603f", @@ -13938,7 +14062,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "79c1aa37-dae3-42ae-b333-d1c105477715", @@ -14080,7 +14205,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "f1e4c7f6-db5c-4035-981f-d35ab4998794", @@ -14126,7 +14252,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5", @@ -14179,7 +14306,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" } ] } diff --git a/airbyte-connector-builder-resources/CDK_VERSION b/airbyte-connector-builder-resources/CDK_VERSION index 969eb252497..edbc6b23886 100644 --- a/airbyte-connector-builder-resources/CDK_VERSION +++ b/airbyte-connector-builder-resources/CDK_VERSION @@ -1 +1 @@ -0.51.2 +0.51.16 diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 99e44391147..664f82c9cbe 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.0 FROM ${BASE_IMAGE} AS connector-builder-server # Set up CDK requirements -ARG CDK_VERSION=0.51.2 +ARG CDK_VERSION=0.51.16 ENV CDK_PYTHON=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/python ENV CDK_ENTRYPOINT ${PYENV_ROOT}/versions/${PYTHON_VERSION}/lib/python3.9/site-packages/airbyte_cdk/connector_builder/main.py # Set up CDK @@ -10,7 +10,7 @@ ENV PIP=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/pip COPY requirements.txt requirements.txt RUN ${PIP} install -r requirements.txt -ARG VERSION=0.50.23 +ARG VERSION=dev ENV APPLICATION airbyte-connector-builder-server ENV VERSION ${VERSION} @@ -23,5 +23,5 @@ ADD airbyte-app.tar /app # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] -LABEL io.airbyte.version=0.50.23 -LABEL io.airbyte.name=airbyte/connector-builder-server \ No newline at end of file +LABEL io.airbyte.version=${VERSION} +LABEL io.airbyte.name=airbyte/connector-builder-server diff --git a/airbyte-connector-builder-server/requirements.in b/airbyte-connector-builder-server/requirements.in index 90e5acfb336..823ef16b513 100644 --- a/airbyte-connector-builder-server/requirements.in +++ b/airbyte-connector-builder-server/requirements.in @@ -1 +1 @@ -airbyte-cdk==0.51.2 +airbyte-cdk==0.51.16 diff --git a/airbyte-connector-builder-server/requirements.txt b/airbyte-connector-builder-server/requirements.txt index 75fa38377d5..b67c8cef3ef 100644 --- a/airbyte-connector-builder-server/requirements.txt +++ b/airbyte-connector-builder-server/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -airbyte-cdk==0.51.2 +airbyte-cdk==0.51.16 # via -r requirements.in airbyte-protocol-models==0.4.0 # via airbyte-cdk @@ -15,7 +15,7 @@ attrs==23.1.0 # requests-cache backoff==2.2.1 # via airbyte-cdk -bracex==2.3.post1 +bracex==2.4 # via wcmatch cachetools==5.3.1 # via airbyte-cdk diff --git a/airbyte-connector-builder-server/src/main/resources/application.yml b/airbyte-connector-builder-server/src/main/resources/application.yml index a576a82a7a4..f7ba3382ee5 100644 --- a/airbyte-connector-builder-server/src/main/resources/application.yml +++ b/airbyte-connector-builder-server/src/main/resources/application.yml @@ -11,6 +11,8 @@ micronaut: netty: access-logger: enabled: ${HTTP_ACCESS_LOG_ENABLED:true} + aggregator: + max-content-length: 52428800 # 50MB endpoints: v1/manifest_template: enable: true diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index f6ba103be2e..fdfe338daa6 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.50.23 +ARG VERSION=dev ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java index 7416ac78dce..ce3b98c47c8 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java @@ -16,6 +16,9 @@ import io.airbyte.container_orchestrator.orchestrator.NormalizationJobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.ReplicationJobOrchestrator; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.ReplicationWorkerFactory; import io.airbyte.workers.internal.state_aggregator.StateAggregatorFactory; @@ -47,6 +50,12 @@ @Factory class ContainerOrchestratorFactory { + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.ORCHESTRATOR); + return MetricClientFactory.getMetricClient(); + } + @Singleton FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle index 68b06adcc76..9a39b0e1a7e 100644 --- a/airbyte-cron/build.gradle +++ b/airbyte-cron/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation project(':airbyte-api') implementation project(':airbyte-analytics') + implementation project(':airbyte-commons') implementation project(':airbyte-commons-micronaut') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-config:config-models') diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java b/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java index db0d5acad6f..9ecdc84d837 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java @@ -4,11 +4,14 @@ package io.airbyte.cron.config; +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.Version; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricEmittingApps; import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Value; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; @@ -37,4 +40,11 @@ public MetricClient metricClient() { return io.airbyte.metrics.lib.MetricClientFactory.getMetricClient(); } + @Singleton + public AirbyteProtocolVersionRange airbyteProtocolVersionRange( + @Value("${airbyte.protocol.min-version}") final String minVersion, + @Value("${airbyte.protocol.max-version}") final String maxVersion) { + return new AirbyteProtocolVersionRange(new Version(minVersion), new Version(maxVersion)); + } + } diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java b/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java index ee624ceb13a..2dc53ab2e72 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java @@ -12,7 +12,9 @@ import io.airbyte.db.factory.DatabaseCheckFactory; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -113,4 +115,10 @@ public JobPersistence jobPersistence(@Named("jobsDatabase") final Database jobDa return new DefaultJobPersistence(jobDatabase); } + @Singleton + @Requires(env = WorkerMode.CONTROL_PLANE) + public MetadataPersistence metadataPersistence(@Named("jobsDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + } diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java index 1e907be004d..cd68389f9aa 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java @@ -9,13 +9,16 @@ import datadog.trace.api.Trace; import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.init.ApplyDefinitionsHelper; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Singleton; +import java.io.IOException; import lombok.extern.slf4j.Slf4j; /** @@ -47,23 +50,11 @@ public DefinitionsUpdater(final ApplyDefinitionsHelper applyDefinitionsHelper, @Trace(operationName = SCHEDULED_TRACE_OPERATION_NAME) @Scheduled(fixedRate = "30s", initialDelay = "1m") - void updateDefinitions() { + void updateDefinitions() throws JsonValidationException, ConfigNotFoundException, IOException { log.info("Updating definitions..."); metricClient.count(OssMetricsRegistry.CRON_JOB_RUN_BY_CRON_TYPE, 1, new MetricAttribute(MetricTags.CRON_TYPE, "definitions_updater")); - - try { - try { - applyDefinitionsHelper.apply(deploymentMode == DeploymentMode.CLOUD); - - log.info("Done applying remote connector definitions"); - } catch (final Exception e) { - log.error("Error while applying remote definitions", e); - } - - } catch (final Exception e) { - log.error("Error when retrieving remote definitions", e); - } - + applyDefinitionsHelper.apply(deploymentMode == DeploymentMode.CLOUD); + log.info("Done applying remote connector definitions"); } } diff --git a/airbyte-cron/src/main/resources/application.yml b/airbyte-cron/src/main/resources/application.yml index 865f75e42ac..50e3fae1929 100644 --- a/airbyte-cron/src/main/resources/application.yml +++ b/airbyte-cron/src/main/resources/application.yml @@ -28,6 +28,9 @@ airbyte: local: docker-mount: ${LOCAL_DOCKER_MOUNT:} root: ${LOCAL_ROOT} + protocol: + min-version: ${AIRBYTE_PROTOCOL_VERSION_MIN:0.0.0} + max-version: ${AIRBYTE_PROTOCOL_VERSION_MAX:0.3.0} role: ${AIRBYTE_ROLE:} temporal: worker: diff --git a/airbyte-data/build.gradle b/airbyte-data/build.gradle new file mode 100644 index 00000000000..6b4aa252f9e --- /dev/null +++ b/airbyte-data/build.gradle @@ -0,0 +1,33 @@ +plugins { + id "io.airbyte.gradle.jvm.lib" +} + +configurations.all { + resolutionStrategy { + // Ensure that the versions defined in deps.toml are used + // instead of versions from transitive dependencies + force libs.flyway.core, libs.s3, libs.aws.java.sdk.s3 + } +} +dependencies { + implementation libs.bundles.jackson + implementation libs.flyway.core + implementation libs.guava + implementation project(':airbyte-commons') + implementation project(':airbyte-config:config-persistence') + implementation project(':airbyte-config:config-models') + implementation project(':airbyte-db:db-lib') + implementation project(':airbyte-db:jooq') + implementation project(':airbyte-json-validation') + implementation libs.airbyte.protocol + + compileOnly libs.lombok + annotationProcessor libs.lombok + testCompileOnly libs.lombok + testAnnotationProcessor libs.lombok + + testImplementation project(':airbyte-test-utils') + testImplementation libs.postgresql + testImplementation libs.platform.testcontainers.postgresql + testImplementation libs.mockito.inline +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java b/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java new file mode 100644 index 00000000000..56b050a9559 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/ActorDefinitionService.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import com.google.common.base.Charsets; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.Version; +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionConfigInjection; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +/** + * This service is responsible for managing the lifecycle of actor definitions. + */ +public interface ActorDefinitionService { + + Set getActorDefinitionIdsInUse() throws IOException; + + Map> getActorDefinitionToProtocolVersionMap() throws IOException; + + Map getActorDefinitionIdsToDefaultVersionsMap() throws IOException; + + int updateActorDefinitionsDockerImageTag(List actorDefinitionIds, String targetImageTag) throws IOException; + + void updateDeclarativeActorDefinition(ActorDefinitionConfigInjection configInjection, ConnectorSpecification spec); + + /** + * Set the ActorDefinitionVersion for a given tag as the default version for the associated actor + * definition. Check docker image tag on the new ADV; if an ADV exists for that tag, set the + * existing ADV for the tag as the default. Otherwise, insert the new ADV and set it as the default. + * + * @param actorDefinitionVersion new actor definition version + * @throws IOException - you never know when you IO + */ + void setActorDefinitionVersionForTagAsDefault(ActorDefinitionVersion actorDefinitionVersion, + List breakingChangesForDefinition) + throws IOException; + + void setActorDefinitionVersionAsDefaultVersion(ActorDefinitionVersion actorDefinitionVersion, + List breakingChangesForDefinition) + throws IOException; + + void updateDefaultVersionIdForActorsOnVersion(UUID previousDefaultVersionId, UUID newDefaultVersionId); + + void updateActorDefinitionDefaultVersionId(UUID actorDefinitionId, UUID versionId); + + void writeActorDefinitionWorkspaceGrant(UUID actorDefinitionId, UUID scopeId, io.airbyte.config.ScopeType scopeType) throws IOException; + + // TODO: The scopeType was an enum but it had jooq imports, so gotta figure out what to do there. + int writeActorDefinitionWorkspaceGrant(UUID actorDefinitionId, UUID scopeId, String scopeType); + + boolean actorDefinitionWorkspaceGrantExists(UUID actorDefinitionId, UUID scopeId, io.airbyte.config.ScopeType scopeType) throws IOException; + + void deleteActorDefinitionWorkspaceGrant(UUID actorDefinitionId, UUID scopeId, io.airbyte.config.ScopeType scopeType) throws IOException; + + ActorDefinitionVersion writeActorDefinitionVersion(ActorDefinitionVersion actorDefinitionVersion) throws IOException; + + Optional getActorDefinitionVersion(UUID actorDefinitionId, String dockerImageTag) throws IOException; + + ActorDefinitionVersion getActorDefinitionVersion(UUID actorDefinitionVersionId) throws IOException, ConfigNotFoundException; + + List listActorDefinitionVersionsForDefinition(UUID actorDefinitionId) throws IOException; + + List getActorDefinitionVersions(List actorDefinitionVersionIds) throws IOException; + + void writeActorDefinitionBreakingChanges(List breakingChanges) throws IOException; + + void setActorDefaultVersion(UUID actorId, UUID actorDefinitionVersionId) throws IOException; + + ActorDefinitionVersion getDefaultVersionForActorDefinitionId(UUID actorDefinitionId); + + /** + * Get an optional ADV for an actor definition's default version. The optional will be empty if the + * defaultVersionId of the actor definition is set to null in the DB. The only time this should be + * the case is if we are in the process of inserting and have already written the source definition, + * but not yet set its default version. + */ + Optional getDefaultVersionForActorDefinitionIdOptional(UUID actorDefinitionId); + + List listBreakingChangesForActorDefinition(UUID actorDefinitionId) throws IOException; + + void setActorDefinitionVersionSupportStates(List actorDefinitionVersionIds, ActorDefinitionVersion.SupportState supportState) + throws IOException; + + List listBreakingChangesForActorDefinitionVersion(ActorDefinitionVersion actorDefinitionVersion) throws IOException; + + List listBreakingChanges() throws IOException; + + /** + * This function generates a hash for the given AirbyteCatalog. + * + * @param airbyteCatalog the catalog to be hashed. + * @return the hash of the catalog. + */ + default String generateCanonicalHash(AirbyteCatalog airbyteCatalog) { + final HashFunction hashFunction = Hashing.murmur3_32_fixed(); + try { + return hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); + } catch (IOException e) { + // TODO: Setup a logger here + // LOGGER.error( + // "Failed to serialize AirbyteCatalog to canonical JSON", e); + return null; + } + } + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/CatalogService.java b/airbyte-data/src/main/java/io/airbyte/data/services/CatalogService.java new file mode 100644 index 00000000000..348012b6842 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/CatalogService.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.ActorCatalog; +import io.airbyte.config.ActorCatalogFetchEvent; +import io.airbyte.config.ActorCatalogWithUpdatedAt; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.protocol.models.AirbyteCatalog; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +/** + * This service is responsible for Catalogs. + */ +public interface CatalogService { + + Map findCatalogByHash(String catalogHash); + + ActorCatalog getActorCatalogById(UUID actorCatalogId) throws IOException, ConfigNotFoundException; + + /** + * Store an Airbyte catalog in DB if it is not present already. + *

+ * Checks in the config DB if the catalog is present already, if so returns it identifier. It is not + * present, it is inserted in DB with a new identifier and that identifier is returned. + * + * @param airbyteCatalog An Airbyte catalog to cache + * @return the db identifier for the cached catalog. + */ + UUID getOrInsertActorCatalog(AirbyteCatalog airbyteCatalog, OffsetDateTime timestamp); + + /** + * This function will be used to gradually migrate the existing data in the database to use the + * canonical json serialization. It will first try to find the catalog using the canonical json + * serialization. If it fails, it will fallback to the old json serialization. + * + * @param airbyteCatalog the catalog to be cached + * @param timestamp - timestamp + * @param writeCatalogInCanonicalJson - should we write the catalog in canonical json + * @return the db identifier for the cached catalog. + */ + UUID getOrInsertCanonicalActorCatalog(AirbyteCatalog airbyteCatalog, OffsetDateTime timestamp, boolean writeCatalogInCanonicalJson); + + UUID lookupCatalogId(String catalogHash, AirbyteCatalog airbyteCatalog); + + UUID insertCatalog(AirbyteCatalog airbyteCatalog, String catalogHash, OffsetDateTime timestamp); + + UUID findAndReturnCatalogId(String catalogHash, AirbyteCatalog airbyteCatalog); + + Optional getActorCatalog(UUID actorId, String actorVersion, String configHash) throws IOException; + + Optional getMostRecentSourceActorCatalog(UUID sourceId) throws IOException; + + Optional getMostRecentActorCatalogForSource(UUID sourceId) throws IOException; + + Optional getMostRecentActorCatalogFetchEventForSource(UUID sourceId) throws IOException; + + UUID writeActorCatalogFetchEvent(AirbyteCatalog catalog, UUID actorId, String connectorVersion, String configurationHash) throws IOException; + + UUID writeCanonicalActorCatalogFetchEvent(AirbyteCatalog catalog, + UUID actorId, + String connectorVersion, + String configurationHash, + boolean writeCatalogInCanonicalJson) + throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/ConnectionService.java b/airbyte-data/src/main/java/io/airbyte/data/services/ConnectionService.java new file mode 100644 index 00000000000..364b43e2970 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/ConnectionService.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.Geography; +import io.airbyte.config.StandardSync; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.data.services.shared.StandardSyncQuery; +import io.airbyte.data.services.shared.StandardSyncsQueryPaginated; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * This service is used to manage connections. + */ +public interface ConnectionService { + + void deleteStandardSync(UUID syncId) throws IOException; + + StandardSync getStandardSync(UUID connectionId) throws JsonValidationException, IOException, ConfigNotFoundException; + + void writeStandardSync(StandardSync standardSync) throws IOException; + + List listStandardSyncs() throws IOException; + + List listStandardSyncsUsingOperation(UUID operationId) throws IOException; + + List listWorkspaceStandardSyncs(UUID workspaceId, boolean includeDeleted) throws IOException; + + List listWorkspaceStandardSyncs(StandardSyncQuery standardSyncQuery) throws IOException; + + Map> listWorkspaceStandardSyncsPaginated(List workspaceIds, boolean includeDeleted, int pageSize, int rowOffset) + throws IOException; + + Map> listWorkspaceStandardSyncsPaginated(StandardSyncsQueryPaginated standardSyncsQueryPaginated) throws IOException; + + List listConnectionsBySource(UUID sourceId, boolean includeDeleted) throws IOException; + + List listConnectionsByActorDefinitionIdAndType(UUID actorDefinitionId, String actorTypeValue, boolean includeDeleted) + throws IOException; + + List getAllStreamsForConnection(UUID connectionId) throws ConfigNotFoundException, IOException; + + ConfiguredAirbyteCatalog getConfiguredCatalogForConnection(UUID connectionId) throws JsonValidationException, ConfigNotFoundException, IOException; + + Geography getGeographyForConnection(UUID connectionId) throws IOException; + + boolean getConnectionHasAlphaOrBetaConnector(UUID connectionId) throws IOException; + + // TODO: This uses jooq stuff so we gotta rework it. + // List getStandardSyncsFromResult( + // Result connectionAndOperationIdsResult, + // List allNeededNotificationConfigurations); + + // TODO: same as above + // Map> getWorkspaceIdToStandardSyncsFromResult( + // Result connectionAndOperationIdsResult, + // List allNeededNotificationConfigurations) +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/ConnectorBuilderService.java b/airbyte-data/src/main/java/io/airbyte/data/services/ConnectorBuilderService.java new file mode 100644 index 00000000000..f893e0bf5f2 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/ConnectorBuilderService.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.config.ActiveDeclarativeManifest; +import io.airbyte.config.ActorDefinitionConfigInjection; +import io.airbyte.config.ConnectorBuilderProject; +import io.airbyte.config.ConnectorBuilderProjectVersionedManifest; +import io.airbyte.config.DeclarativeManifest; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.io.IOException; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * This service is responsible for managing the lifecycle of connector builder projects. + */ +public interface ConnectorBuilderService { + + ConnectorBuilderProject getConnectorBuilderProject(UUID builderProjectId, boolean fetchManifestDraft) throws IOException, ConfigNotFoundException; + + ConnectorBuilderProjectVersionedManifest getVersionedConnectorBuilderProject(UUID builderProjectId, Long version) + throws ConfigNotFoundException, IOException; + + Stream getConnectorBuilderProjectsByWorkspace(UUID workspaceId) throws IOException; + + ConnectorBuilderProjectVersionedManifest buildConnectorBuilderProjectVersionedManifest(); + + boolean deleteBuilderProject(UUID builderProjectId) throws IOException; + + void writeBuilderProjectDraft(UUID projectId, UUID workspaceId, String name, JsonNode manifestDraft) throws IOException; + + void deleteBuilderProjectDraft(UUID projectId) throws IOException; + + void deleteManifestDraftForActorDefinition(UUID actorDefinitionId, UUID workspaceId) throws IOException; + + void updateBuilderProjectAndActorDefinition(UUID projectId, UUID workspaceId, String name, JsonNode manifestDraft, UUID actorDefinitionId) + throws IOException; + + void assignActorDefinitionToConnectorBuilderProject(UUID builderProjectId, UUID actorDefinitionId) throws IOException; + + void createDeclarativeManifestAsActiveVersion(DeclarativeManifest declarativeManifest, + ActorDefinitionConfigInjection configInjection, + ConnectorSpecification connectorSpecification) + throws IOException; + + void upsertActiveDeclarativeManifest(ActiveDeclarativeManifest activeDeclarativeManifest); + + void setDeclarativeSourceActiveVersion(UUID sourceDefinitionId, + Long version, + ActorDefinitionConfigInjection configInjection, + ConnectorSpecification connectorSpecification) + throws IOException; + + Stream getActorDefinitionConfigInjections(UUID actorDefinitionId) throws IOException; + + void writeActorDefinitionConfigInjectionForPath(ActorDefinitionConfigInjection actorDefinitionConfigInjection) throws IOException; + + void insertDeclarativeManifest(DeclarativeManifest declarativeManifest) throws IOException; + + void insertActiveDeclarativeManifest(DeclarativeManifest declarativeManifest) throws IOException; + + Stream getDeclarativeManifestsByActorDefinitionId(UUID actorDefinitionId) throws IOException; + + DeclarativeManifest getDeclarativeManifestByActorDefinitionIdAndVersion(UUID actorDefinitionId, long version) + throws IOException, ConfigNotFoundException; + + DeclarativeManifest getCurrentlyActiveDeclarativeManifestsByActorDefinitionId(UUID actorDefinitionId) throws IOException, ConfigNotFoundException; + + Stream getActorDefinitionIdsWithActiveDeclarativeManifest() throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/DestinationService.java b/airbyte-data/src/main/java/io/airbyte/data/services/DestinationService.java new file mode 100644 index 00000000000..31daf93ba7f --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/DestinationService.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.data.services.shared.DestinationAndDefinition; +import io.airbyte.data.services.shared.ResourcesQueryPaginated; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * This service is used to interact with destinations. + */ +public interface DestinationService { + + Stream destDefQuery(Optional destDefId, boolean includeTombstone) throws IOException; + + StandardDestinationDefinition getStandardDestinationDefinition(UUID destinationDefinitionId) + throws JsonValidationException, IOException, ConfigNotFoundException; + + StandardDestinationDefinition getDestinationDefinitionFromDestination(UUID destinationId); + + StandardDestinationDefinition getDestinationDefinitionFromConnection(UUID connectionId); + + List listStandardDestinationDefinitions(boolean includeTombstone) throws IOException; + + List listPublicDestinationDefinitions(boolean includeTombstone) throws IOException; + + List listGrantedDestinationDefinitions(UUID workspaceId, boolean includeTombstones) throws IOException; + + List> listGrantableDestinationDefinitions(UUID workspaceId, boolean includeTombstones) + throws IOException; + + void updateStandardDestinationDefinition(StandardDestinationDefinition destinationDefinition) + throws IOException, JsonValidationException, ConfigNotFoundException; + + void writeDestinationDefinitionAndDefaultVersion(StandardDestinationDefinition destinationDefinition, + ActorDefinitionVersion actorDefinitionVersion, + List breakingChangesForDefinition) + throws IOException; + + void writeDestinationDefinitionAndDefaultVersion(StandardDestinationDefinition destinationDefinition, ActorDefinitionVersion actorDefinitionVersion) + throws IOException; + + void writeCustomDestinationDefinitionAndDefaultVersion(StandardDestinationDefinition destinationDefinition, + ActorDefinitionVersion defaultVersion, + UUID scopeId, + io.airbyte.config.ScopeType scopeType) + throws IOException; + + Stream listDestinationQuery(Optional configId); + + DestinationConnection getDestinationConnection(UUID destinationId) throws JsonValidationException, IOException, ConfigNotFoundException; + + void writeDestinationConnectionNoSecrets(DestinationConnection partialDestination) throws IOException; + + void writeDestinationConnection(List configs); + + List listDestinationConnection() throws IOException; + + List listWorkspaceDestinationConnection(UUID workspaceId) throws IOException; + + List listWorkspacesDestinationConnections(ResourcesQueryPaginated resourcesQueryPaginated) throws IOException; + + List listDestinationsForDefinition(UUID definitionId) throws IOException; + + List getDestinationAndDefinitionsFromDestinationIds(List destinationIds) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/HealthCheckService.java b/airbyte-data/src/main/java/io/airbyte/data/services/HealthCheckService.java new file mode 100644 index 00000000000..8868bf7b2e2 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/HealthCheckService.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +/** + * Service to check the health of the server. + */ +public interface HealthCheckService { + + boolean healthCheck(); + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/NotificationService.java b/airbyte-data/src/main/java/io/airbyte/data/services/NotificationService.java new file mode 100644 index 00000000000..9183baa2170 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/NotificationService.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; + +/** + * Notification Service. + */ +public interface NotificationService { + + // TODO: the List is what was here originally, but that is jooq + // specific. + List getNotificationConfigurationByConnectionIds(List connnectionIds) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/OAuthService.java b/airbyte-data/src/main/java/io/airbyte/data/services/OAuthService.java new file mode 100644 index 00000000000..d59989cbf65 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/OAuthService.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * OAuth Service. + */ +public interface OAuthService { + + Stream listSourceOauthParamQuery(Optional configId) throws IOException; + + Optional getSourceOAuthParamByDefinitionIdOptional(UUID workspaceId, UUID sourceDefinitionId) throws IOException; + + void writeSourceOAuthParam(SourceOAuthParameter sourceOAuthParameter) throws IOException; + + void writeSourceOauthParameter(List configs); + + List listSourceOAuthParam() throws JsonValidationException, IOException; + + /** + * List destination oauth param query. If configId is present only returns the config for that oauth + * parameter id. if not present then lists all. + * + * @param configId oauth parameter id optional. + * @return stream of destination oauth params + * @throws IOException if there is an issue while interacting with db. + */ + Stream listDestinationOauthParamQuery(Optional configId) throws IOException; + + Optional getDestinationOAuthParamByDefinitionIdOptional(UUID workspaceId, UUID destinationDefinitionId) + throws IOException; + + void writeDestinationOAuthParam(DestinationOAuthParameter destinationOAuthParameter) throws IOException; + + void writeDestinationOauthParameter(List configs); + + List listDestinationOAuthParam() + throws JsonValidationException, IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/OperationService.java b/airbyte-data/src/main/java/io/airbyte/data/services/OperationService.java new file mode 100644 index 00000000000..40bb9663324 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/OperationService.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * Operation Service. + */ +public interface OperationService { + + Stream listStandardSyncOperationQuery(Optional configId) throws IOException; + + StandardSyncOperation getStandardSyncOperation(UUID operationId) throws JsonValidationException, IOException, ConfigNotFoundException; + + void writeStandardSyncOperation(StandardSyncOperation standardSyncOperation) throws IOException; + + void writeStandardSyncOperation(List configs); + + List listStandardSyncOperations() throws IOException; + + void updateConnectionOperationIds(UUID connectionId, Set newOperationIds) throws IOException; + + void deleteStandardSyncOperation(UUID standardSyncOperationId) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/OrganizationService.java b/airbyte-data/src/main/java/io/airbyte/data/services/OrganizationService.java new file mode 100644 index 00000000000..42af39d44b4 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/OrganizationService.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.Organization; +import io.airbyte.data.services.shared.ResourcesByOrganizationQueryPaginated; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * A service that manages organizations. + */ +public interface OrganizationService { + + Optional getOrganization(UUID organizationId) throws IOException; + + void writeOrganization(Organization organization) throws IOException; + + List listOrganizations() throws IOException; + + Stream listOrganizationQuery(Optional organizationId) throws IOException; + + List listOrganizationsPaginated(ResourcesByOrganizationQueryPaginated resourcesByOrganizationQueryPaginated) throws IOException; + + Optional getOrganizationIdFromWorkspaceId(final UUID scopeId) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/SourceService.java b/airbyte-data/src/main/java/io/airbyte/data/services/SourceService.java new file mode 100644 index 00000000000..19fcf6842e5 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/SourceService.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.data.services.shared.ResourcesQueryPaginated; +import io.airbyte.data.services.shared.SourceAndDefinition; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * A service that manages sources. + */ +public interface SourceService { + + StandardSourceDefinition getStandardSourceDefinition(UUID sourceDefinitionId) throws JsonValidationException, IOException, ConfigNotFoundException; + + StandardSourceDefinition getSourceDefinitionFromSource(UUID sourceId); + + StandardSourceDefinition getSourceDefinitionFromConnection(UUID connectionId); + + List listStandardSourceDefinitions(boolean includeTombstone) throws IOException; + + Stream sourceDefQuery(Optional sourceDefId, boolean includeTombstone) throws IOException; + + List listPublicSourceDefinitions(boolean includeTombstone) throws IOException; + + List listGrantedSourceDefinitions(UUID workspaceId, boolean includeTombstones) throws IOException; + + List> listGrantableSourceDefinitions(UUID workspaceId, boolean includeTombstones) throws IOException; + + void updateStandardSourceDefinition(StandardSourceDefinition sourceDefinition) throws IOException, JsonValidationException, ConfigNotFoundException; + + void writeSourceDefinitionAndDefaultVersion(StandardSourceDefinition sourceDefinition, + ActorDefinitionVersion actorDefinitionVersion, + List breakingChangesForDefinition) + throws IOException; + + void writeSourceDefinitionAndDefaultVersion(StandardSourceDefinition sourceDefinition, ActorDefinitionVersion actorDefinitionVersion) + throws IOException; + + void writeCustomSourceDefinitionAndDefaultVersion(StandardSourceDefinition sourceDefinition, + ActorDefinitionVersion defaultVersion, + UUID scopeId, + io.airbyte.config.ScopeType scopeType) + throws IOException; + + Stream listSourceQuery(Optional configId) throws IOException; + + SourceConnection getSourceConnection(UUID sourceId) throws JsonValidationException, ConfigNotFoundException, IOException; + + void writeSourceConnectionNoSecrets(SourceConnection partialSource) throws IOException; + + void writeSourceConnection(List configs); + + boolean deleteSource(UUID sourceId) throws JsonValidationException, ConfigNotFoundException, IOException; + + List listSourceConnection() throws IOException; + + List listWorkspaceSourceConnection(UUID workspaceId) throws IOException; + + List listWorkspacesSourceConnections(ResourcesQueryPaginated resourcesQueryPaginated) throws IOException; + + List listSourcesForDefinition(UUID definitionId) throws IOException; + + List getSourceAndDefinitionsFromSourceIds(List sourceIds) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/WorkspaceService.java b/airbyte-data/src/main/java/io/airbyte/data/services/WorkspaceService.java new file mode 100644 index 00000000000..743c8e177a6 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/WorkspaceService.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.config.Geography; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WorkspaceServiceAccount; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.data.services.shared.ResourcesQueryPaginated; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +/** + * A service that manages workspaces. + */ +public interface WorkspaceService { + + StandardWorkspace getStandardWorkspaceNoSecrets(UUID workspaceId, boolean includeTombstone) + throws JsonValidationException, IOException, ConfigNotFoundException; + + Optional getWorkspaceBySlugOptional(String slug, boolean includeTombstone) throws IOException; + + StandardWorkspace getWorkspaceBySlug(String slug, boolean includeTombstone) throws IOException, ConfigNotFoundException; + + List listStandardWorkspaces(boolean includeTombstone) throws IOException; + + List listAllWorkspacesPaginated(ResourcesQueryPaginated resourcesQueryPaginated) throws IOException; + + Stream listWorkspaceQuery(Optional workspaceId, boolean includeTombstone) throws IOException; + + List listStandardWorkspacesPaginated(ResourcesQueryPaginated resourcesQueryPaginated) throws IOException; + + StandardWorkspace getStandardWorkspaceFromConnection(UUID connectionId, boolean isTombstone) throws ConfigNotFoundException; + + void writeStandardWorkspaceNoSecrets(StandardWorkspace workspace) throws JsonValidationException, IOException; + + void setFeedback(UUID workspaceId) throws IOException; + + boolean workspaceCanUseDefinition(UUID actorDefinitionId, UUID workspaceId) throws IOException; + + boolean workspaceCanUseCustomDefinition(UUID actorDefinitionId, UUID workspaceId) throws IOException; + + List listActiveWorkspacesByMostRecentlyRunningJobs(int timeWindowInHours) throws IOException; + + int countConnectionsForWorkspace(UUID workspaceId) throws IOException; + + int countSourcesForWorkspace(UUID workspaceId) throws IOException; + + int countDestinationsForWorkspace(UUID workspaceId) throws IOException; + + WorkspaceServiceAccount getWorkspaceServiceAccountNoSecrets(UUID workspaceId) throws IOException, ConfigNotFoundException; + + void writeWorkspaceServiceAccountNoSecrets(WorkspaceServiceAccount workspaceServiceAccount) throws IOException; + + Geography getGeographyForWorkspace(UUID workspaceId) throws IOException; + + boolean getWorkspaceHasAlphaOrBetaConnector(UUID workspaceId) throws IOException; + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/WorkspaceServiceJooqImpl.java b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/WorkspaceServiceJooqImpl.java new file mode 100644 index 00000000000..907a124b235 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/impls/jooq/WorkspaceServiceJooqImpl.java @@ -0,0 +1,685 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.impls.jooq; + +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_WORKSPACE_GRANT; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION_OPERATION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.NOTIFICATION_CONFIGURATION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.SCHEMA_MANAGEMENT; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE_SERVICE_ACCOUNT; +import static io.airbyte.db.instance.jobs.jooq.generated.Tables.JOBS; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.noCondition; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.SQLDataType.VARCHAR; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.ConfigWithMetadata; +import io.airbyte.config.Geography; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WorkspaceServiceAccount; +import io.airbyte.config.helpers.ScheduleHelpers; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.DbConverter; +import io.airbyte.data.services.WorkspaceService; +import io.airbyte.data.services.shared.ResourcesQueryPaginated; +import io.airbyte.db.Database; +import io.airbyte.db.ExceptionWrappingDatabase; +import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType; +import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; +import io.airbyte.db.instance.configs.jooq.generated.enums.ScopeType; +import io.airbyte.db.instance.configs.jooq.generated.enums.StatusType; +import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; +import io.airbyte.validation.json.JsonValidationException; +import jakarta.inject.Singleton; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.JoinType; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.SelectJoinStep; + +@Singleton +public class WorkspaceServiceJooqImpl implements WorkspaceService { + + private final ExceptionWrappingDatabase database; + + @VisibleForTesting + public WorkspaceServiceJooqImpl(final Database database) { + this.database = new ExceptionWrappingDatabase(database); + } + + /** + * Get workspace. + * + * @param workspaceId workspace id + * @param includeTombstone include tombestoned workspace + * @return workspace + * @throws JsonValidationException - throws if returned sources are invalid + * @throws IOException - you never know when you IO + * @throws ConfigNotFoundException - throws if no source with that id can be found. + */ + @Override + public StandardWorkspace getStandardWorkspaceNoSecrets(final UUID workspaceId, final boolean includeTombstone) + throws JsonValidationException, IOException, ConfigNotFoundException { + return listWorkspaceQuery(Optional.of(workspaceId), includeTombstone) + .findFirst() + .orElseThrow(() -> new ConfigNotFoundException(ConfigSchema.STANDARD_WORKSPACE, workspaceId)); + } + + /** + * Get workspace from slug. + * + * @param slug to use to find the workspace + * @param includeTombstone include tombestoned workspace + * @return workspace, if present. + * @throws IOException - you never know when you IO + */ + @Override + public Optional getWorkspaceBySlugOptional(final String slug, final boolean includeTombstone) + throws IOException { + final Result result; + if (includeTombstone) { + result = database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.SLUG.eq(slug))).fetch(); + } else { + result = database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.SLUG.eq(slug)).andNot(WORKSPACE.TOMBSTONE)).fetch(); + } + + return result.stream().findFirst().map(DbConverter::buildStandardWorkspace); + } + + /** + * Get workspace from slug. + * + * @param slug to use to find the workspace + * @param includeTombstone include tombestoned workspace + * @return workspace + * @throws IOException - you never know when you IO + * @throws ConfigNotFoundException - throws if no source with that id can be found. + */ + @Override + public StandardWorkspace getWorkspaceBySlug(final String slug, final boolean includeTombstone) throws IOException, ConfigNotFoundException { + return getWorkspaceBySlugOptional(slug, includeTombstone).orElseThrow(() -> new ConfigNotFoundException(ConfigSchema.STANDARD_WORKSPACE, slug)); + } + + /** + * List workspaces. + * + * @param includeTombstone include tombstoned workspaces + * @return workspaces + * @throws IOException - you never know when you IO + */ + @Override + public List listStandardWorkspaces(final boolean includeTombstone) throws IOException { + return listWorkspaceQuery(Optional.empty(), includeTombstone).toList(); + } + + /** + * List ALL workspaces (paginated) with some filtering. + * + * @param resourcesQueryPaginated - contains all the information we need to paginate + * @return A List of StandardWorkspace objects + * @throws IOException you never know when you IO + */ + @Override + public List listAllWorkspacesPaginated(final ResourcesQueryPaginated resourcesQueryPaginated) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(resourcesQueryPaginated.includeDeleted() ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .and(resourcesQueryPaginated.nameContains() != null ? WORKSPACE.NAME.contains(resourcesQueryPaginated.nameContains()) : noCondition()) + .limit(resourcesQueryPaginated.pageSize()) + .offset(resourcesQueryPaginated.rowOffset()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + @Override + public Stream listWorkspaceQuery(final Optional workspaceId, final boolean includeTombstone) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(includeTombstone ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .and(workspaceId.map(WORKSPACE.ID::eq).orElse(noCondition())) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace); + } + + /** + * List workspaces (paginated). + * + * @param resourcesQueryPaginated - contains all the information we need to paginate + * @return A List of StandardWorkspace objectjs + * @throws IOException you never know when you IO + */ + @Override + public List listStandardWorkspacesPaginated(final ResourcesQueryPaginated resourcesQueryPaginated) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(resourcesQueryPaginated.includeDeleted() ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .and(WORKSPACE.ID.in(resourcesQueryPaginated.workspaceIds())) + .limit(resourcesQueryPaginated.pageSize()) + .offset(resourcesQueryPaginated.rowOffset()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * Get workspace for a connection. + * + * @param connectionId connection id + * @param isTombstone include tombstoned workspaces + * @return workspace to which the connection belongs + */ + @Override + public StandardWorkspace getStandardWorkspaceFromConnection(final UUID connectionId, final boolean isTombstone) throws ConfigNotFoundException { + try { + final StandardSync sync = getStandardSync(connectionId); + final SourceConnection source = getSourceConnection(sync.getSourceId()); + return getStandardWorkspaceNoSecrets(source.getWorkspaceId(), isTombstone); + } catch (final ConfigNotFoundException e) { + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + /** + * MUST NOT ACCEPT SECRETS - Should only be called from { @link SecretsRepositoryWriter }. + *

+ * Write a StandardWorkspace to the database. + * + * @param workspace - The configuration of the workspace + * @throws JsonValidationException - throws is the workspace is invalid + * @throws IOException - you never know when you IO + */ + @Override + public void writeStandardWorkspaceNoSecrets(final StandardWorkspace workspace) throws JsonValidationException, IOException { + database.transaction(ctx -> { + final OffsetDateTime timestamp = OffsetDateTime.now(); + final boolean isExistingConfig = ctx.fetchExists(select() + .from(WORKSPACE) + .where(WORKSPACE.ID.eq(workspace.getWorkspaceId()))); + + if (isExistingConfig) { + ctx.update(WORKSPACE) + .set(WORKSPACE.ID, workspace.getWorkspaceId()) + .set(WORKSPACE.CUSTOMER_ID, workspace.getCustomerId()) + .set(WORKSPACE.NAME, workspace.getName()) + .set(WORKSPACE.SLUG, workspace.getSlug()) + .set(WORKSPACE.EMAIL, workspace.getEmail()) + .set(WORKSPACE.INITIAL_SETUP_COMPLETE, workspace.getInitialSetupComplete()) + .set(WORKSPACE.ANONYMOUS_DATA_COLLECTION, workspace.getAnonymousDataCollection()) + .set(WORKSPACE.SEND_NEWSLETTER, workspace.getNews()) + .set(WORKSPACE.SEND_SECURITY_UPDATES, workspace.getSecurityUpdates()) + .set(WORKSPACE.DISPLAY_SETUP_WIZARD, workspace.getDisplaySetupWizard()) + .set(WORKSPACE.TOMBSTONE, workspace.getTombstone() != null && workspace.getTombstone()) + .set(WORKSPACE.NOTIFICATIONS, JSONB.valueOf(Jsons.serialize(workspace.getNotifications()))) + .set(WORKSPACE.NOTIFICATION_SETTINGS, JSONB.valueOf(Jsons.serialize(workspace.getNotificationSettings()))) + .set(WORKSPACE.FIRST_SYNC_COMPLETE, workspace.getFirstCompletedSync()) + .set(WORKSPACE.FEEDBACK_COMPLETE, workspace.getFeedbackDone()) + .set(WORKSPACE.GEOGRAPHY, Enums.toEnum( + workspace.getDefaultGeography().value(), + io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) + .set(WORKSPACE.UPDATED_AT, timestamp) + .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, workspace.getWebhookOperationConfigs() == null ? null + : JSONB.valueOf(Jsons.serialize(workspace.getWebhookOperationConfigs()))) + .set(WORKSPACE.ORGANIZATION_ID, workspace.getOrganizationId()) + .where(WORKSPACE.ID.eq(workspace.getWorkspaceId())) + .execute(); + } else { + ctx.insertInto(WORKSPACE) + .set(WORKSPACE.ID, workspace.getWorkspaceId()) + .set(WORKSPACE.CUSTOMER_ID, workspace.getCustomerId()) + .set(WORKSPACE.NAME, workspace.getName()) + .set(WORKSPACE.SLUG, workspace.getSlug()) + .set(WORKSPACE.EMAIL, workspace.getEmail()) + .set(WORKSPACE.INITIAL_SETUP_COMPLETE, workspace.getInitialSetupComplete()) + .set(WORKSPACE.ANONYMOUS_DATA_COLLECTION, workspace.getAnonymousDataCollection()) + .set(WORKSPACE.SEND_NEWSLETTER, workspace.getNews()) + .set(WORKSPACE.SEND_SECURITY_UPDATES, workspace.getSecurityUpdates()) + .set(WORKSPACE.DISPLAY_SETUP_WIZARD, workspace.getDisplaySetupWizard()) + .set(WORKSPACE.TOMBSTONE, workspace.getTombstone() != null && workspace.getTombstone()) + .set(WORKSPACE.NOTIFICATIONS, JSONB.valueOf(Jsons.serialize(workspace.getNotifications()))) + .set(WORKSPACE.NOTIFICATION_SETTINGS, JSONB.valueOf(Jsons.serialize(workspace.getNotificationSettings()))) + .set(WORKSPACE.FIRST_SYNC_COMPLETE, workspace.getFirstCompletedSync()) + .set(WORKSPACE.FEEDBACK_COMPLETE, workspace.getFeedbackDone()) + .set(WORKSPACE.CREATED_AT, timestamp) + .set(WORKSPACE.UPDATED_AT, timestamp) + .set(WORKSPACE.GEOGRAPHY, Enums.toEnum( + workspace.getDefaultGeography().value(), + io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType.class).orElseThrow()) + .set(WORKSPACE.ORGANIZATION_ID, workspace.getOrganizationId()) + .set(WORKSPACE.WEBHOOK_OPERATION_CONFIGS, workspace.getWebhookOperationConfigs() == null ? null + : JSONB.valueOf(Jsons.serialize(workspace.getWebhookOperationConfigs()))) + .execute(); + } + return null; + + }); + } + + /** + * Set user feedback on workspace. + * + * @param workspaceId workspace id. + * @throws IOException - you never know when you IO + */ + @Override + public void setFeedback(final UUID workspaceId) throws IOException { + database.query(ctx -> ctx.update(WORKSPACE).set(WORKSPACE.FEEDBACK_COMPLETE, true).where(WORKSPACE.ID.eq(workspaceId)).execute()); + } + + /** + * Test if workspace id has access to a connector definition. + * + * @param actorDefinitionId actor definition id + * @param workspaceId id of the workspace + * @return true, if the workspace has access. otherwise, false. + * @throws IOException - you never know when you IO + */ + @Override + public boolean workspaceCanUseDefinition(final UUID actorDefinitionId, final UUID workspaceId) throws IOException { + return scopeCanUseDefinition(actorDefinitionId, workspaceId, ScopeType.workspace.toString()); + } + + /** + * Test if workspace is has access to a custom connector definition. + * + * @param actorDefinitionId custom actor definition id + * @param workspaceId workspace id + * @return true, if the workspace has access. otherwise, false. + * @throws IOException - you never know when you IO + */ + @Override + public boolean workspaceCanUseCustomDefinition(final UUID actorDefinitionId, final UUID workspaceId) throws IOException { + final Result records = actorDefinitionsJoinedWithGrants( + workspaceId, + ScopeType.workspace, + JoinType.JOIN, + ACTOR_DEFINITION.ID.eq(actorDefinitionId), + ACTOR_DEFINITION.CUSTOM.eq(true)); + return records.isNotEmpty(); + } + + /** + * List active workspace IDs with most recently running jobs within a given time window (in hours). + * + * @param timeWindowInHours - integer, e.g. 24, 48, etc + * @return list of workspace IDs + * @throws IOException - failed to query data + */ + @Override + public List listActiveWorkspacesByMostRecentlyRunningJobs(final int timeWindowInHours) throws IOException { + final Result> records = database.query(ctx -> ctx.selectDistinct(ACTOR.WORKSPACE_ID) + .from(ACTOR) + .join(WORKSPACE) + .on(ACTOR.WORKSPACE_ID.eq(WORKSPACE.ID)) + .join(CONNECTION) + .on(CONNECTION.SOURCE_ID.eq(ACTOR.ID)) + .join(JOBS) + .on(CONNECTION.ID.cast(VARCHAR(255)).eq(JOBS.SCOPE)) + .where(JOBS.UPDATED_AT.greaterOrEqual(OffsetDateTime.now().minusHours(timeWindowInHours))) + .and(WORKSPACE.TOMBSTONE.isFalse()) + .fetch()); + return records.stream().map(record -> record.get(ACTOR.WORKSPACE_ID)).collect(Collectors.toList()); + } + + /** + * Count connections in workspace. + * + * @param workspaceId workspace id + * @return number of connections in workspace + * @throws IOException if there is an issue while interacting with db. + */ + @Override + public int countConnectionsForWorkspace(final UUID workspaceId) throws IOException { + return database.query(ctx -> ctx.selectCount() + .from(CONNECTION) + .join(ACTOR).on(CONNECTION.SOURCE_ID.eq(ACTOR.ID)) + .where(ACTOR.WORKSPACE_ID.eq(workspaceId)) + .and(CONNECTION.STATUS.notEqual(StatusType.deprecated)) + .andNot(ACTOR.TOMBSTONE)).fetchOne().into(int.class); + } + + /** + * Count sources in workspace. + * + * @param workspaceId workspace id + * @return number of sources in workspace + * @throws IOException if there is an issue while interacting with db. + */ + @Override + public int countSourcesForWorkspace(final UUID workspaceId) throws IOException { + return database.query(ctx -> ctx.selectCount() + .from(ACTOR) + .where(ACTOR.WORKSPACE_ID.equal(workspaceId)) + .and(ACTOR.ACTOR_TYPE.eq(ActorType.source)) + .andNot(ACTOR.TOMBSTONE)).fetchOne().into(int.class); + } + + /** + * Count destinations in workspace. + * + * @param workspaceId workspace id + * @return number of destinations in workspace + * @throws IOException if there is an issue while interacting with db. + */ + @Override + public int countDestinationsForWorkspace(final UUID workspaceId) throws IOException { + return database.query(ctx -> ctx.selectCount() + .from(ACTOR) + .where(ACTOR.WORKSPACE_ID.equal(workspaceId)) + .and(ACTOR.ACTOR_TYPE.eq(ActorType.destination)) + .andNot(ACTOR.TOMBSTONE)).fetchOne().into(int.class); + } + + /** + * Get workspace service account without secrets. + * + * @param workspaceId workspace id + * @return workspace service account + * @throws ConfigNotFoundException if the config does not exist + * @throws IOException if there is an issue while interacting with db. + */ + @Override + public WorkspaceServiceAccount getWorkspaceServiceAccountNoSecrets(final UUID workspaceId) throws IOException, ConfigNotFoundException { + // breaking the pattern of doing a list query, because we never want to list this resource without + // scoping by workspace id. + return database.query(ctx -> ctx.select(asterisk()).from(WORKSPACE_SERVICE_ACCOUNT) + .where(WORKSPACE_SERVICE_ACCOUNT.WORKSPACE_ID.eq(workspaceId)) + .fetch()) + .map(DbConverter::buildWorkspaceServiceAccount) + .stream() + .findFirst() + .orElseThrow(() -> new ConfigNotFoundException(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceId)); + } + + /** + * Write workspace service account with no secrets. + * + * @param workspaceServiceAccount workspace service account + * @throws IOException if there is an issue while interacting with db. + */ + @Override + public void writeWorkspaceServiceAccountNoSecrets(final WorkspaceServiceAccount workspaceServiceAccount) throws IOException { + database.transaction(ctx -> { + writeWorkspaceServiceAccount(Collections.singletonList(workspaceServiceAccount), ctx); + return null; + }); + } + + /** + * Get geography for a workspace. + * + * @param workspaceId workspace id + * @return geography + * @throws IOException exception while interacting with the db + */ + @Override + public Geography getGeographyForWorkspace(final UUID workspaceId) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.GEOGRAPHY) + .from(WORKSPACE) + .where(WORKSPACE.ID.eq(workspaceId)) + .limit(1)) + .fetchOneInto(Geography.class); + } + + /** + * Specialized query for efficiently determining eligibility for the Free Connector Program. If a + * workspace has at least one Alpha or Beta connector, users of that workspace will be prompted to + * sign up for the program. This check is performed on nearly every page load so the query needs to + * be as efficient as possible. + *

+ * This should only be used for efficiently determining eligibility for the Free Connector Program. + * Anything that involves billing should instead use the ActorDefinitionVersionHelper to determine + * the ReleaseStages. + * + * @param workspaceId ID of the workspace to check connectors for + * @return boolean indicating if an alpha or beta connector exists within the workspace + */ + @Override + public boolean getWorkspaceHasAlphaOrBetaConnector(final UUID workspaceId) throws IOException { + final Condition releaseStageAlphaOrBeta = ACTOR_DEFINITION_VERSION.RELEASE_STAGE.eq(ReleaseStage.alpha) + .or(ACTOR_DEFINITION_VERSION.RELEASE_STAGE.eq(ReleaseStage.beta)); + + final Integer countResult = database.query(ctx -> ctx.selectCount() + .from(ACTOR) + .join(ACTOR_DEFINITION).on(ACTOR_DEFINITION.ID.eq(ACTOR.ACTOR_DEFINITION_ID)) + .join(ACTOR_DEFINITION_VERSION).on(ACTOR_DEFINITION_VERSION.ID.eq(ACTOR_DEFINITION.DEFAULT_VERSION_ID)) + .where(ACTOR.WORKSPACE_ID.eq(workspaceId)) + .and(ACTOR.TOMBSTONE.notEqual(true)) + .and(releaseStageAlphaOrBeta)) + .fetchOneInto(Integer.class); + + return countResult > 0; + } + + /** + * Write workspace service account. + * + * @param configs list of workspace service account + * @param ctx database context + */ + private void writeWorkspaceServiceAccount(final List configs, final DSLContext ctx) { + final OffsetDateTime timestamp = OffsetDateTime.now(); + configs.forEach((workspaceServiceAccount) -> { + final boolean isExistingConfig = ctx.fetchExists(select() + .from(WORKSPACE_SERVICE_ACCOUNT) + .where(WORKSPACE_SERVICE_ACCOUNT.WORKSPACE_ID.eq(workspaceServiceAccount.getWorkspaceId()))); + + if (isExistingConfig) { + ctx.update(WORKSPACE_SERVICE_ACCOUNT) + .set(WORKSPACE_SERVICE_ACCOUNT.WORKSPACE_ID, workspaceServiceAccount.getWorkspaceId()) + .set(WORKSPACE_SERVICE_ACCOUNT.SERVICE_ACCOUNT_ID, workspaceServiceAccount.getServiceAccountId()) + .set(WORKSPACE_SERVICE_ACCOUNT.SERVICE_ACCOUNT_EMAIL, workspaceServiceAccount.getServiceAccountEmail()) + .set(WORKSPACE_SERVICE_ACCOUNT.JSON_CREDENTIAL, JSONB.valueOf(Jsons.serialize(workspaceServiceAccount.getJsonCredential()))) + .set(WORKSPACE_SERVICE_ACCOUNT.HMAC_KEY, JSONB.valueOf(Jsons.serialize(workspaceServiceAccount.getHmacKey()))) + .set(WORKSPACE_SERVICE_ACCOUNT.UPDATED_AT, timestamp) + .where(WORKSPACE_SERVICE_ACCOUNT.WORKSPACE_ID.eq(workspaceServiceAccount.getWorkspaceId())) + .execute(); + } else { + ctx.insertInto(WORKSPACE_SERVICE_ACCOUNT) + .set(WORKSPACE_SERVICE_ACCOUNT.WORKSPACE_ID, workspaceServiceAccount.getWorkspaceId()) + .set(WORKSPACE_SERVICE_ACCOUNT.SERVICE_ACCOUNT_ID, workspaceServiceAccount.getServiceAccountId()) + .set(WORKSPACE_SERVICE_ACCOUNT.SERVICE_ACCOUNT_EMAIL, workspaceServiceAccount.getServiceAccountEmail()) + .set(WORKSPACE_SERVICE_ACCOUNT.JSON_CREDENTIAL, JSONB.valueOf(Jsons.serialize(workspaceServiceAccount.getJsonCredential()))) + .set(WORKSPACE_SERVICE_ACCOUNT.HMAC_KEY, JSONB.valueOf(Jsons.serialize(workspaceServiceAccount.getHmacKey()))) + .set(WORKSPACE_SERVICE_ACCOUNT.CREATED_AT, timestamp) + .set(WORKSPACE_SERVICE_ACCOUNT.UPDATED_AT, timestamp) + .execute(); + } + }); + } + + /** + * Get connection. + * + * @param connectionId connection id + * @return connection + * @throws JsonValidationException if the workspace is or contains invalid json + * @throws ConfigNotFoundException if the config does not exist + * @throws IOException if there is an issue while interacting with db. + */ + @VisibleForTesting + public StandardSync getStandardSync(final UUID connectionId) throws JsonValidationException, IOException, ConfigNotFoundException { + return getStandardSyncWithMetadata(connectionId).getConfig(); + } + + private ConfigWithMetadata getStandardSyncWithMetadata(final UUID connectionId) throws IOException, ConfigNotFoundException { + final List> result = listStandardSyncWithMetadata(Optional.of(connectionId)); + + final boolean foundMoreThanOneConfig = result.size() > 1; + if (result.isEmpty()) { + throw new ConfigNotFoundException(ConfigSchema.STANDARD_SYNC, connectionId.toString()); + } else if (foundMoreThanOneConfig) { + throw new IllegalStateException(String.format("Multiple %s configs found for ID %s: %s", ConfigSchema.STANDARD_SYNC, connectionId, result)); + } + return result.get(0); + } + + private List> listStandardSyncWithMetadata(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(CONNECTION.asterisk(), + SCHEMA_MANAGEMENT.AUTO_PROPAGATION_STATUS) + .from(CONNECTION) + // The schema management can be non-existent for a connection id, thus we need to do a left join + .leftJoin(SCHEMA_MANAGEMENT).on(SCHEMA_MANAGEMENT.CONNECTION_ID.eq(CONNECTION.ID)); + if (configId.isPresent()) { + return query.where(CONNECTION.ID.eq(configId.get())).fetch(); + } + return query.fetch(); + }); + + final List> standardSyncs = new ArrayList<>(); + for (final Record record : result) { + final List notificationConfigurationRecords = database.query(ctx -> { + if (configId.isPresent()) { + return ctx.selectFrom(NOTIFICATION_CONFIGURATION) + .where(NOTIFICATION_CONFIGURATION.CONNECTION_ID.eq(configId.get())) + .fetch(); + } else { + return ctx.selectFrom(NOTIFICATION_CONFIGURATION) + .fetch(); + } + }); + + final StandardSync standardSync = + DbConverter.buildStandardSync(record, connectionOperationIds(record.get(CONNECTION.ID)), notificationConfigurationRecords); + if (ScheduleHelpers.isScheduleTypeMismatch(standardSync)) { + throw new RuntimeException("unexpected schedule type mismatch"); + } + standardSyncs.add(new ConfigWithMetadata<>( + record.get(CONNECTION.ID).toString(), + ConfigSchema.STANDARD_SYNC.name(), + record.get(CONNECTION.CREATED_AT).toInstant(), + record.get(CONNECTION.UPDATED_AT).toInstant(), + standardSync)); + } + return standardSyncs; + } + + private List connectionOperationIds(final UUID connectionId) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(CONNECTION_OPERATION) + .where(CONNECTION_OPERATION.CONNECTION_ID.eq(connectionId)) + .fetch()); + + final List ids = new ArrayList<>(); + for (final Record record : result) { + ids.add(record.get(CONNECTION_OPERATION.OPERATION_ID)); + } + + return ids; + } + + /** + * Returns source with a given id. Does not contain secrets. To hydrate with secrets see { @link + * SecretsRepositoryReader#getSourceConnectionWithSecrets(final UUID sourceId) }. + * + * @param sourceId - id of source to fetch. + * @return sources + * @throws JsonValidationException - throws if returned sources are invalid + * @throws IOException - you never know when you IO + * @throws ConfigNotFoundException - throws if no source with that id can be found. + */ + @VisibleForTesting + public SourceConnection getSourceConnection(final UUID sourceId) throws JsonValidationException, ConfigNotFoundException, IOException { + return listSourceQuery(Optional.of(sourceId)) + .findFirst() + .orElseThrow(() -> new ConfigNotFoundException(ConfigSchema.SOURCE_CONNECTION, sourceId)); + } + + /** + * Test if workspace or organization id has access to a connector definition. + * + * @param actorDefinitionId actor definition id + * @param scopeId id of the workspace or organization + * @param scopeType enum of workspace or organization + * @return true, if the workspace or organization has access. otherwise, false. + * @throws IOException - you never know when you IO + */ + private boolean scopeCanUseDefinition(final UUID actorDefinitionId, final UUID scopeId, final String scopeType) throws IOException { + final Result records = actorDefinitionsJoinedWithGrants( + scopeId, + ScopeType.valueOf(scopeType), + JoinType.LEFT_OUTER_JOIN, + ACTOR_DEFINITION.ID.eq(actorDefinitionId), + ACTOR_DEFINITION.PUBLIC.eq(true).or(ACTOR_DEFINITION_WORKSPACE_GRANT.ACTOR_DEFINITION_ID.eq(actorDefinitionId))); + return records.isNotEmpty(); + } + + private Stream listSourceQuery(final Optional configId) throws IOException { + final Result result = database.query(ctx -> { + final SelectJoinStep query = ctx.select(asterisk()).from(ACTOR); + if (configId.isPresent()) { + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.source), ACTOR.ID.eq(configId.get())).fetch(); + } + return query.where(ACTOR.ACTOR_TYPE.eq(ActorType.source)).fetch(); + }); + + return result.map(DbConverter::buildSourceConnection).stream(); + } + + private Result actorDefinitionsJoinedWithGrants(final UUID scopeId, + final ScopeType scopeType, + final JoinType joinType, + final Condition... conditions) + throws IOException { + Condition scopeConditional = ACTOR_DEFINITION_WORKSPACE_GRANT.SCOPE_TYPE.eq(ScopeType.valueOf(scopeType.toString())).and( + ACTOR_DEFINITION_WORKSPACE_GRANT.SCOPE_ID.eq(scopeId)); + + // if scope type is workspace, get organization id as well and add that into OR conditional + if (scopeType == ScopeType.workspace) { + final Optional organizationId = getOrganizationIdFromWorkspaceId(scopeId); + if (organizationId.isPresent()) { + scopeConditional = scopeConditional.or(ACTOR_DEFINITION_WORKSPACE_GRANT.SCOPE_TYPE.eq(ScopeType.organization).and( + ACTOR_DEFINITION_WORKSPACE_GRANT.SCOPE_ID.eq(organizationId.get()))); + } + } + + final Condition finalScopeConditional = scopeConditional; + return database.query(ctx -> ctx.select(asterisk()).from(ACTOR_DEFINITION) + .join(ACTOR_DEFINITION_WORKSPACE_GRANT, joinType) + .on(ACTOR_DEFINITION.ID.eq(ACTOR_DEFINITION_WORKSPACE_GRANT.ACTOR_DEFINITION_ID).and(finalScopeConditional)) + .where(conditions) + .fetch()); + } + + private Optional getOrganizationIdFromWorkspaceId(final UUID scopeId) throws IOException { + final Optional> optionalRecord = database.query(ctx -> ctx.select(WORKSPACE.ORGANIZATION_ID).from(WORKSPACE) + .where(WORKSPACE.ID.eq(scopeId)).fetchOptional()); + return optionalRecord.map(Record1::value1); + } + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/DestinationAndDefinition.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/DestinationAndDefinition.java new file mode 100644 index 00000000000..39ccddae865 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/DestinationAndDefinition.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.StandardDestinationDefinition; + +/** + * A pair of a destination connection and its associated definition. + * + * @param destination Destination. + * @param definition Destination definition. + */ +public record DestinationAndDefinition(DestinationConnection destination, StandardDestinationDefinition definition) { + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesByOrganizationQueryPaginated.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesByOrganizationQueryPaginated.java new file mode 100644 index 00000000000..ffa691b8444 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesByOrganizationQueryPaginated.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import jakarta.annotation.Nonnull; +import java.util.UUID; + +/** + * Query object for paginated querying of resource in an organization. + * + * @param organizationId organization to fetch resources for + * @param includeDeleted include tombstoned resources + * @param pageSize limit + * @param rowOffset offset + */ +public record ResourcesByOrganizationQueryPaginated( + @Nonnull UUID organizationId, + boolean includeDeleted, + int pageSize, + int rowOffset) { + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesQueryPaginated.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesQueryPaginated.java new file mode 100644 index 00000000000..c407f7a5d4c --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/ResourcesQueryPaginated.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.UUID; + +/** + * Query object for paginated querying of sources/destinations in multiple workspaces. + * + * @param workspaceIds workspaces to fetch resources for + * @param includeDeleted include tombstoned resources + * @param pageSize limit + * @param rowOffset offset + * @param nameContains string to search name contains by + */ +public record ResourcesQueryPaginated( + @Nonnull List workspaceIds, + boolean includeDeleted, + int pageSize, + int rowOffset, + String nameContains) { + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/SourceAndDefinition.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/SourceAndDefinition.java new file mode 100644 index 00000000000..64cfe501e03 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/SourceAndDefinition.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardSourceDefinition; + +/** + * Pair of source and its associated definition. + *

+ * Data-carrier records to hold combined result of query for a Source or Destination and its + * corresponding Definition. This enables the API layer to process combined information about a + * Source/Destination/Definition pair without requiring two separate queries and in-memory join + * operation, because the config models are grouped immediately in the repository layer. + * + * @param source source + * @param definition its corresponding definition + */ +public record SourceAndDefinition(SourceConnection source, StandardSourceDefinition definition) { + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncQuery.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncQuery.java new file mode 100644 index 00000000000..f241bfd641d --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncQuery.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.UUID; + +/** + * Query object for querying connections for a workspace. + * + * @param workspaceId workspace to fetch connections for + * @param sourceId fetch connections with this source id + * @param destinationId fetch connections with this destination id + * @param includeDeleted include tombstoned connections + */ +public record StandardSyncQuery(@Nonnull UUID workspaceId, List sourceId, List destinationId, boolean includeDeleted) { + +} diff --git a/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncsQueryPaginated.java b/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncsQueryPaginated.java new file mode 100644 index 00000000000..50e3276e747 --- /dev/null +++ b/airbyte-data/src/main/java/io/airbyte/data/services/shared/StandardSyncsQueryPaginated.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services.shared; + +import jakarta.annotation.Nonnull; +import java.util.List; +import java.util.UUID; + +/** + * Query object for paginated querying of connections in multiple workspaces. + * + * @param workspaceIds workspaces to fetch connections for + * @param sourceId fetch connections with this source id + * @param destinationId fetch connections with this destination id + * @param includeDeleted include tombstoned connections + * @param pageSize limit + * @param rowOffset offset + */ +public record StandardSyncsQueryPaginated( + @Nonnull List workspaceIds, + List sourceId, + List destinationId, + boolean includeDeleted, + int pageSize, + int rowOffset) { + +} diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/BaseConfigDatabaseTest.java b/airbyte-data/src/test/java/io/airbyte/data/services/BaseConfigDatabaseTest.java new file mode 100644 index 00000000000..133953afe0e --- /dev/null +++ b/airbyte-data/src/test/java/io/airbyte/data/services/BaseConfigDatabaseTest.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.init.DatabaseInitializationException; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.ConfigsDatabaseTestProvider; +import io.airbyte.db.instance.test.TestDatabaseProviders; +import io.airbyte.test.utils.Databases; +import java.io.IOException; +import java.sql.SQLException; +import javax.sql.DataSource; +import org.flywaydb.core.Flyway; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * This class exists to abstract away the lifecycle of the test container database and the config + * database schema. This is ALL it intends to do. Any additional functionality belongs somewhere + * else. It is useful for test suites that need to interact directly with the database. + * + * This class sets up a test container database and runs the config database migrations against it + * to provide the most up-to-date schema. + * + * What this class is NOT designed to do: + *

    + *
  • test migration behavior, only should be used to test query behavior against the current + * schema.
    • + *
  • expose database details -- if you are attempting to expose container, dataSource, dslContext, + * something is wrong.
    • + *
  • add test fixtures or helpers--do NOT put "generic" resource helper methods (e.g. + * createTestSource())
    • + *
+ * + * This comment is emphatically worded, because it is tempting to add things to this class. It has + * already happened in 3 previous iterations, and each time it takes multiple engineering days to + * fix it. + * + * Usage: + *
    + *
  • Extend: Extend this class. By doing so, it will automatically create the test container db + * and run migrations against it at the start of the test suite (@BeforeAll).
    • + *
  • Use database: As part of the @BeforeAll the database field is set. This is the only field + * that the extending class can access. It's lifecycle is fully managed by this class.
    • + *
  • Reset schema: To reset the database in between tests, call truncateAllTables() as part + * of @BeforeEach. This is the only method that this class exposes externally. It is exposed in such + * a way, because most test suites need to declare their own @BeforeEach, so it is easier for them + * to simply call this method there, then trying to apply a more complex inheritance scheme.
    • + *
+ * + * Note: truncateAllTables() works by truncating each table in the db, if you add a new table, you + * will need to add it to that method for it work as expected. + */ +@SuppressWarnings({"PMD.MutableStaticState", "PMD.SignatureDeclareThrowsException"}) +class BaseConfigDatabaseTest { + + static Database database; + + // keep these private, do not expose outside this class! + private static PostgreSQLContainer container; + private static DataSource dataSource; + private static DSLContext dslContext; + + /** + * Create db test container, sets up java database resources, and runs migrations. Should not be + * called externally. It is not private because junit cannot access private methods. + * + * @throws DatabaseInitializationException - db fails to initialize + * @throws IOException - failure when interacting with db. + */ + @BeforeAll + static void dbSetup() throws DatabaseInitializationException, IOException { + createDbContainer(); + setDb(); + migrateDb(); + } + + /** + * Close all resources (container, data source, dsl context, database). Should not be called + * externally. It is not private because junit cannot access private methods. + * + * @throws Exception - exception while closing resources + */ + @AfterAll + static void dbDown() throws Exception { + DataSourceFactory.close(dataSource); + container.close(); + } + + /** + * Truncates tables to reset them. Designed to be used in between tests. + * + * Note: NEW TABLES -- When a new table is added to the db, it will need to be added here. + * + * @throws SQLException - failure in truncate query. + */ + static void truncateAllTables() throws SQLException { + database.query(ctx -> ctx + .execute( + """ + TRUNCATE TABLE + active_declarative_manifest, + actor, + actor_catalog, + actor_catalog_fetch_event, + actor_definition, + actor_definition_breaking_change, + actor_definition_version, + actor_definition_workspace_grant, + actor_definition_config_injection, + actor_oauth_parameter, + connection, + connection_operation, + connector_builder_project, + declarative_manifest, + notification_configuration, + operation, + organization, + permission, + schema_management, + state, + stream_reset, + \"user\", + user_invitation, + sso_config, + organization_email_domain, + workspace, + workspace_service_account + """)); + } + + private static void createDbContainer() { + container = new PostgreSQLContainer<>("postgres:13-alpine") + .withDatabaseName("airbyte") + .withUsername("docker") + .withPassword("docker"); + container.start(); + } + + private static void setDb() throws DatabaseInitializationException, IOException { + dataSource = Databases.createDataSource(container); + dslContext = DSLContextFactory.create(dataSource, SQLDialect.POSTGRES); + final TestDatabaseProviders databaseProviders = new TestDatabaseProviders(dataSource, dslContext); + database = databaseProviders.createNewConfigsDatabase(); + databaseProviders.createNewJobsDatabase(); + } + + private static void migrateDb() throws IOException, DatabaseInitializationException { + final Flyway flyway = FlywayFactory.create( + dataSource, + StreamResetPersistenceTest.class.getName(), + ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + new ConfigsDatabaseTestProvider(dslContext, flyway).create(true); + } + +} diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/MockData.java b/airbyte-data/src/test/java/io/airbyte/data/services/MockData.java new file mode 100644 index 00000000000..5ad7d68efbe --- /dev/null +++ b/airbyte-data/src/test/java/io/airbyte/data/services/MockData.java @@ -0,0 +1,935 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.version.Version; +import io.airbyte.config.ActiveDeclarativeManifest; +import io.airbyte.config.ActorCatalog; +import io.airbyte.config.ActorCatalogFetchEvent; +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionConfigInjection; +import io.airbyte.config.ActorDefinitionResourceRequirements; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DeclarativeManifest; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.FieldSelectionData; +import io.airbyte.config.Geography; +import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; +import io.airbyte.config.Notification; +import io.airbyte.config.Notification.NotificationType; +import io.airbyte.config.OperatorDbt; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.OperatorNormalization.Option; +import io.airbyte.config.OperatorWebhook; +import io.airbyte.config.Organization; +import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.ResourceRequirements; +import io.airbyte.config.Schedule; +import io.airbyte.config.Schedule.TimeUnit; +import io.airbyte.config.SlackNotificationConfiguration; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.NonBreakingChangesPreference; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.StandardSyncOperation; +import io.airbyte.config.StandardSyncOperation.OperatorType; +import io.airbyte.config.StandardSyncState; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; +import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.WebhookConfig; +import io.airbyte.config.WebhookOperationConfigs; +import io.airbyte.config.WorkspaceServiceAccount; +import io.airbyte.protocol.models.AdvancedAuth; +import io.airbyte.protocol.models.AdvancedAuth.AuthFlowType; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.SyncMode; +import java.net.URI; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import lombok.Data; + +@SuppressWarnings({"MissingJavadocMethod", "MissingJavadocType", "LineLength"}) +public class MockData { + + public static final UUID WORKSPACE_ID_1 = UUID.randomUUID(); + public static final UUID WORKSPACE_ID_2 = UUID.randomUUID(); + private static final UUID WORKSPACE_ID_3 = UUID.randomUUID(); + private static final UUID WORKSPACE_CUSTOMER_ID = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_3 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID_4 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_VERSION_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_VERSION_ID_2 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_VERSION_ID_3 = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_VERSION_ID_4 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_3 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID_4 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_VERSION_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_VERSION_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_VERSION_ID_3 = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_VERSION_ID_4 = UUID.randomUUID(); + public static final UUID SOURCE_ID_1 = UUID.randomUUID(); + public static final UUID SOURCE_ID_2 = UUID.randomUUID(); + private static final UUID SOURCE_ID_3 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_1 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_2 = UUID.randomUUID(); + public static final UUID DESTINATION_ID_3 = UUID.randomUUID(); + private static final UUID OPERATION_ID_1 = UUID.randomUUID(); + private static final UUID OPERATION_ID_2 = UUID.randomUUID(); + private static final UUID OPERATION_ID_3 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_1 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_2 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_3 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_4 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_5 = UUID.randomUUID(); + private static final UUID CONNECTION_ID_6 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID SOURCE_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_1 = UUID.randomUUID(); + private static final UUID DESTINATION_OAUTH_PARAMETER_ID_2 = UUID.randomUUID(); + public static final UUID ACTOR_CATALOG_ID_1 = UUID.randomUUID(); + private static final UUID ACTOR_CATALOG_ID_2 = UUID.randomUUID(); + public static final UUID ACTOR_CATALOG_ID_3 = UUID.randomUUID(); + private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_1 = UUID.randomUUID(); + private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_2 = UUID.randomUUID(); + private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_3 = UUID.randomUUID(); + public static final long DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES = 10800; + public static final Supplier MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER = () -> DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES; + // User + static final UUID CREATOR_USER_ID_1 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_2 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_3 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_4 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_5 = UUID.randomUUID(); + + // Permission + static final UUID PERMISSION_ID_1 = UUID.randomUUID(); + static final UUID PERMISSION_ID_2 = UUID.randomUUID(); + static final UUID PERMISSION_ID_3 = UUID.randomUUID(); + static final UUID PERMISSION_ID_4 = UUID.randomUUID(); + + static final UUID PERMISSION_ID_5 = UUID.randomUUID(); + static final UUID PERMISSION_ID_6 = UUID.randomUUID(); + static final UUID PERMISSION_ID_7 = UUID.randomUUID(); + + static final UUID ORGANIZATION_ID_1 = UUID.randomUUID(); + static final UUID ORGANIZATION_ID_2 = UUID.randomUUID(); + static final UUID ORGANIZATION_ID_3 = UUID.randomUUID(); + + public static final String MOCK_SERVICE_ACCOUNT_1 = "{\n" + + " \"type\" : \"service_account\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + + public static final String MOCK_SERVICE_ACCOUNT_2 = "{\n" + + " \"type\" : \"service_account-2\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + + public static final JsonNode HMAC_SECRET_PAYLOAD_1 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + public static final JsonNode HMAC_SECRET_PAYLOAD_2 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEX", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeX"))); + + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); + + private static final String CONNECTION_SPECIFICATION = "{\"name\":\"John\", \"age\":30, \"car\":null}"; + private static final UUID OPERATION_ID_4 = UUID.randomUUID(); + private static final UUID WEBHOOK_CONFIG_ID = UUID.randomUUID(); + private static final String WEBHOOK_OPERATION_EXECUTION_URL = "test-webhook-url"; + private static final String WEBHOOK_OPERATION_EXECUTION_BODY = "test-webhook-body"; + public static final String CONFIG_HASH = "1394"; + public static final String CONNECTOR_VERSION = "1.2.0"; + + public static List users() { + final User user1 = new User() + .withUserId(CREATOR_USER_ID_1) + .withName("user-1") + .withAuthUserId(CREATOR_USER_ID_1.toString()) + .withAuthProvider(AuthProvider.GOOGLE_IDENTITY_PLATFORM) + .withDefaultWorkspaceId(WORKSPACE_ID_1) + .withStatus(User.Status.DISABLED) + .withCompanyName("company-1") + .withEmail("user-1@whatever.com") + .withNews(true); + + final User user2 = new User() + .withUserId(CREATOR_USER_ID_2) + .withName("user-2") + .withAuthUserId(CREATOR_USER_ID_2.toString()) + .withAuthProvider(AuthProvider.GOOGLE_IDENTITY_PLATFORM) + .withDefaultWorkspaceId(WORKSPACE_ID_2) + .withStatus(User.Status.INVITED) + .withCompanyName("company-2") + .withEmail("user-2@whatever.com") + .withNews(false); + + final User user3 = new User() + .withUserId(CREATOR_USER_ID_3) + .withName("user-3") + .withAuthUserId(CREATOR_USER_ID_3.toString()) + .withAuthProvider(AuthProvider.GOOGLE_IDENTITY_PLATFORM) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("company-3") + .withEmail("user-3@whatever.com") + .withNews(true); + + final User user4 = new User() + .withUserId(CREATOR_USER_ID_4) + .withName("user-4") + .withAuthUserId(CREATOR_USER_ID_4.toString()) + .withAuthProvider(AuthProvider.GOOGLE_IDENTITY_PLATFORM) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("company-4") + .withEmail("user-4@whatever.com") + .withNews(true); + + final User user5 = new User() + .withUserId(CREATOR_USER_ID_5) + .withName("user-5") + .withAuthUserId(CREATOR_USER_ID_5.toString()) + .withAuthProvider(AuthProvider.KEYCLOAK) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("company-5") + .withEmail("user-5@whatever.com") + .withNews(true); + + return Arrays.asList(user1, user2, user3, user4, user5); + } + + public static List permissions() { + final Permission permission1 = new Permission() + .withPermissionId(PERMISSION_ID_1) + .withUserId(CREATOR_USER_ID_1) + .withWorkspaceId(WORKSPACE_ID_1) + .withPermissionType(PermissionType.INSTANCE_ADMIN); + + final Permission permission2 = new Permission() + .withPermissionId(PERMISSION_ID_2) + .withUserId(CREATOR_USER_ID_2) + .withWorkspaceId(WORKSPACE_ID_2) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission3 = new Permission() + .withPermissionId(PERMISSION_ID_3) + .withUserId(CREATOR_USER_ID_3) + .withWorkspaceId(null) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission4 = new Permission() + .withPermissionId(PERMISSION_ID_4) + .withUserId(CREATOR_USER_ID_1) + .withWorkspaceId(WORKSPACE_ID_1) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission5 = new Permission() + .withPermissionId(PERMISSION_ID_5) + .withUserId(CREATOR_USER_ID_4) + .withOrganizationId(ORGANIZATION_ID_1) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN); + + final Permission permission6 = new Permission() + .withPermissionId(PERMISSION_ID_6) + .withUserId(CREATOR_USER_ID_5) + .withWorkspaceId(WORKSPACE_ID_2) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission7 = new Permission() + .withPermissionId(PERMISSION_ID_7) + .withUserId(CREATOR_USER_ID_5) + .withOrganizationId(ORGANIZATION_ID_2) + .withPermissionType(PermissionType.ORGANIZATION_READER); + + return Arrays.asList(permission1, permission2, permission3, permission4, permission5, permission6, permission7); + } + + public static List organizations() { + final Organization organization1 = + new Organization().withOrganizationId(ORGANIZATION_ID_1).withName("organization-1").withEmail("email@email.com"); + final Organization organization2 = + new Organization().withOrganizationId(ORGANIZATION_ID_2).withName("organization-2").withEmail("email2@email.com"); + final Organization organization3 = + new Organization().withOrganizationId(ORGANIZATION_ID_3).withName("organization-3").withEmail("emai3l@email.com"); + return Arrays.asList(organization1, organization2, organization3); + } + + public static List standardWorkspaces() { + final Notification notification = new Notification() + .withNotificationType(NotificationType.SLACK) + .withSendOnFailure(true) + .withSendOnSuccess(true) + .withSlackConfiguration(new SlackNotificationConfiguration().withWebhook("webhook-url")); + + final StandardWorkspace workspace1 = new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID_1) + .withCustomerId(WORKSPACE_CUSTOMER_ID) + .withName("test-workspace") + .withSlug("random-string") + .withEmail("abc@xyz.com") + .withInitialSetupComplete(true) + .withAnonymousDataCollection(true) + .withNews(true) + .withSecurityUpdates(true) + .withDisplaySetupWizard(true) + .withTombstone(false) + .withNotifications(Collections.singletonList(notification)) + .withFirstCompletedSync(true) + .withFeedbackDone(true) + .withDefaultGeography(Geography.US) + .withWebhookOperationConfigs(Jsons.jsonNode( + new WebhookOperationConfigs().withWebhookConfigs(List.of(new WebhookConfig().withId(WEBHOOK_CONFIG_ID).withName("name"))))); + + final StandardWorkspace workspace2 = new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID_2) + .withName("Another Workspace") + .withSlug("another-workspace") + .withInitialSetupComplete(true) + .withTombstone(false) + .withDefaultGeography(Geography.AUTO); + + final StandardWorkspace workspace3 = new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID_3) + .withName("Tombstoned") + .withSlug("tombstoned") + .withInitialSetupComplete(true) + .withTombstone(true) + .withDefaultGeography(Geography.AUTO); + + return Arrays.asList(workspace1, workspace2, workspace3); + } + + public static StandardSourceDefinition publicSourceDefinition() { + return new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_1) + .withSourceType(SourceType.API) + .withName("random-source-1") + .withIcon("icon-1") + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + public static StandardSourceDefinition grantableSourceDefinition1() { + return new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_2) + .withSourceType(SourceType.DATABASE) + .withName("random-source-2") + .withIcon("icon-2") + .withTombstone(false) + .withPublic(false) + .withCustom(false) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + public static StandardSourceDefinition grantableSourceDefinition2() { + return new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_3) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_3) + .withSourceType(SourceType.DATABASE) + .withName("random-source-3") + .withIcon("icon-3") + .withTombstone(false) + .withPublic(false) + .withCustom(false) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + public static StandardSourceDefinition customSourceDefinition() { + return new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID_4) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_4) + .withSourceType(SourceType.DATABASE) + .withName("random-source-4") + .withIcon("icon-4") + .withTombstone(false) + .withPublic(false) + .withCustom(true) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + public static ActorDefinitionVersion actorDefinitionVersion() { + return new ActorDefinitionVersion() + .withDockerImageTag("0.0.1") + .withDockerRepository("repository-4") + .withSpec(connectorSpecification()) + .withProtocolVersion("0.2.0"); + } + + public static ActorDefinitionBreakingChange actorDefinitionBreakingChange(final String version) { + return new ActorDefinitionBreakingChange() + .withVersion(new Version(version)) + .withMessage("This is a breaking change for version " + version) + .withMigrationDocumentationUrl("https://docs.airbyte.com/migration#" + version) + .withUpgradeDeadline("2020-01-01"); + } + + public static List standardSourceDefinitions() { + return Arrays.asList( + publicSourceDefinition(), + grantableSourceDefinition1(), + grantableSourceDefinition2(), + customSourceDefinition()); + } + + public static ConnectorSpecification connectorSpecification() { + return new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withDocumentationUrl(URI.create("whatever")) + .withAdvancedAuth(new AdvancedAuth().withAuthFlowType(AuthFlowType.OAUTH_2_0)) + .withChangelogUrl(URI.create("whatever")) + .withSupportedDestinationSyncModes(Arrays.asList(DestinationSyncMode.APPEND, DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND_DEDUP)) + .withSupportsDBT(true) + .withSupportsIncremental(true) + .withSupportsNormalization(true); + } + + public static StandardDestinationDefinition publicDestinationDefinition() { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_1) + .withName("random-destination-1") + .withIcon("icon-3") + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); + } + + public static StandardDestinationDefinition grantableDestinationDefinition1() { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_2) + .withName("random-destination-2") + .withIcon("icon-4") + .withTombstone(false) + .withPublic(false) + .withCustom(false); + } + + public static StandardDestinationDefinition grantableDestinationDefinition2() { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_3) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_3) + .withName("random-destination-3") + .withIcon("icon-3") + .withTombstone(false) + .withPublic(false) + .withCustom(false); + } + + public static StandardDestinationDefinition customDestinationDefinition() { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_4) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_4) + .withName("random-destination-4") + .withIcon("icon-4") + .withTombstone(false) + .withPublic(false) + .withCustom(true); + } + + public static List standardDestinationDefinitions() { + return Arrays.asList( + publicDestinationDefinition(), + grantableDestinationDefinition1(), + grantableDestinationDefinition2(), + customDestinationDefinition()); + } + + public static List sourceConnections() { + final SourceConnection sourceConnection1 = new SourceConnection() + .withName("source-1") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_1) + .withWorkspaceId(WORKSPACE_ID_1) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) + .withSourceId(SOURCE_ID_1); + final SourceConnection sourceConnection2 = new SourceConnection() + .withName("source-2") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_2) + .withWorkspaceId(WORKSPACE_ID_1) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) + .withSourceId(SOURCE_ID_2); + final SourceConnection sourceConnection3 = new SourceConnection() + .withName("source-3") + .withTombstone(false) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withDefaultVersionId(SOURCE_DEFINITION_VERSION_ID_1) + .withWorkspaceId(WORKSPACE_ID_2) + .withConfiguration(Jsons.emptyObject()) + .withSourceId(SOURCE_ID_3); + return Arrays.asList(sourceConnection1, sourceConnection2, sourceConnection3); + } + + public static List destinationConnections() { + final DestinationConnection destinationConnection1 = new DestinationConnection() + .withName("destination-1") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_1) + .withWorkspaceId(WORKSPACE_ID_1) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) + .withDestinationId(DESTINATION_ID_1); + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withName("destination-2") + .withTombstone(false) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_2) + .withWorkspaceId(WORKSPACE_ID_1) + .withConfiguration(Jsons.deserialize(CONNECTION_SPECIFICATION)) + .withDestinationId(DESTINATION_ID_2); + final DestinationConnection destinationConnection3 = new DestinationConnection() + .withName("destination-3") + .withTombstone(true) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withDefaultVersionId(DESTINATION_DEFINITION_VERSION_ID_2) + .withWorkspaceId(WORKSPACE_ID_2) + .withConfiguration(Jsons.emptyObject()) + .withDestinationId(DESTINATION_ID_3); + return Arrays.asList(destinationConnection1, destinationConnection2, destinationConnection3); + } + + public static List sourceOauthParameters() { + final SourceOAuthParameter sourceOAuthParameter1 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withWorkspaceId(WORKSPACE_ID_1) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_1) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_1); + final SourceOAuthParameter sourceOAuthParameter2 = new SourceOAuthParameter() + .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withWorkspaceId(WORKSPACE_ID_1) + .withSourceDefinitionId(SOURCE_DEFINITION_ID_2) + .withOauthParameterId(SOURCE_OAUTH_PARAMETER_ID_2); + return Arrays.asList(sourceOAuthParameter1, sourceOAuthParameter2); + } + + public static List destinationOauthParameters() { + final DestinationOAuthParameter destinationOAuthParameter1 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withWorkspaceId(WORKSPACE_ID_1) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_1) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_1); + final DestinationOAuthParameter destinationOAuthParameter2 = new DestinationOAuthParameter() + .withConfiguration(Jsons.jsonNode(CONNECTION_SPECIFICATION)) + .withWorkspaceId(WORKSPACE_ID_1) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID_2) + .withOauthParameterId(DESTINATION_OAUTH_PARAMETER_ID_2); + return Arrays.asList(destinationOAuthParameter1, destinationOAuthParameter2); + } + + public static List standardSyncOperations() { + final OperatorDbt operatorDbt = new OperatorDbt() + .withDbtArguments("dbt-arguments") + .withDockerImage("image-tag") + .withGitRepoBranch("git-repo-branch") + .withGitRepoUrl("git-repo-url"); + final StandardSyncOperation standardSyncOperation1 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_1) + .withWorkspaceId(WORKSPACE_ID_1) + .withOperatorDbt(operatorDbt) + .withOperatorNormalization(null) + .withOperatorType(OperatorType.DBT); + final StandardSyncOperation standardSyncOperation2 = new StandardSyncOperation() + .withName("operation-1") + .withTombstone(false) + .withOperationId(OPERATION_ID_2) + .withWorkspaceId(WORKSPACE_ID_1) + .withOperatorDbt(null) + .withOperatorNormalization(new OperatorNormalization().withOption(Option.BASIC)) + .withOperatorType(OperatorType.NORMALIZATION); + final StandardSyncOperation standardSyncOperation3 = new StandardSyncOperation() + .withName("operation-3") + .withTombstone(false) + .withOperationId(OPERATION_ID_3) + .withWorkspaceId(WORKSPACE_ID_2) + .withOperatorDbt(null) + .withOperatorNormalization(new OperatorNormalization().withOption(Option.BASIC)) + .withOperatorType(OperatorType.NORMALIZATION); + final StandardSyncOperation standardSyncOperation4 = new StandardSyncOperation() + .withName("webhook-operation") + .withTombstone(false) + .withOperationId(OPERATION_ID_4) + .withWorkspaceId(WORKSPACE_ID_1) + .withOperatorType(OperatorType.WEBHOOK) + .withOperatorDbt(null) + .withOperatorNormalization(null) + .withOperatorWebhook( + new OperatorWebhook() + .withWebhookConfigId(WEBHOOK_CONFIG_ID) + .withExecutionUrl(WEBHOOK_OPERATION_EXECUTION_URL) + .withExecutionBody(WEBHOOK_OPERATION_EXECUTION_BODY)); + return Arrays.asList(standardSyncOperation1, standardSyncOperation2, standardSyncOperation3, standardSyncOperation4); + } + + public static List standardSyncs() { + final ResourceRequirements resourceRequirements = new ResourceRequirements() + .withCpuRequest("1") + .withCpuLimit("1") + .withMemoryRequest("1") + .withMemoryLimit("1"); + final Schedule schedule = new Schedule().withTimeUnit(TimeUnit.DAYS).withUnits(1L); + final StandardSync standardSync1 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_1) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withFieldSelectionData(new FieldSelectionData().withAdditionalProperty("foo", true)) + .withName("standard-sync-1") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + final StandardSync standardSync2 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_2) + .withSourceId(SOURCE_ID_1) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-2") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + final StandardSync standardSync3 = new StandardSync() + .withOperationIds(Arrays.asList(OPERATION_ID_1, OPERATION_ID_2)) + .withConnectionId(CONNECTION_ID_3) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_1) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-3") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.DESTINATION) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + final StandardSync standardSync4 = new StandardSync() + .withOperationIds(Collections.emptyList()) + .withConnectionId(CONNECTION_ID_4) + .withSourceId(SOURCE_ID_2) + .withDestinationId(DESTINATION_ID_2) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-4") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.DEPRECATED) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + final StandardSync standardSync5 = new StandardSync() + .withOperationIds(List.of(OPERATION_ID_3)) + .withConnectionId(CONNECTION_ID_5) + .withSourceId(SOURCE_ID_3) + .withDestinationId(DESTINATION_ID_3) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-5") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.ACTIVE) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + final StandardSync standardSync6 = new StandardSync() + .withOperationIds(List.of()) + .withConnectionId(CONNECTION_ID_6) + .withSourceId(SOURCE_ID_3) + .withDestinationId(DESTINATION_ID_3) + .withCatalog(getConfiguredCatalog()) + .withName("standard-sync-6") + .withManual(true) + .withNamespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT) + .withNamespaceFormat("") + .withPrefix("") + .withResourceRequirements(resourceRequirements) + .withStatus(Status.DEPRECATED) + .withSchedule(schedule) + .withGeography(Geography.AUTO) + .withBreakingChange(false) + .withNonBreakingChangesPreference(NonBreakingChangesPreference.IGNORE) + .withNotifySchemaChanges(false) + .withNotifySchemaChangesByEmail(false); + + return Arrays.asList(standardSync1, standardSync2, standardSync3, standardSync4, standardSync5, standardSync6); + } + + private static ConfiguredAirbyteCatalog getConfiguredCatalog() { + final AirbyteCatalog catalog = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + "models", + "models_schema", + io.airbyte.protocol.models.Field.of("id", JsonSchemaType.NUMBER), + io.airbyte.protocol.models.Field.of("make_id", JsonSchemaType.NUMBER), + io.airbyte.protocol.models.Field.of("model", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + return CatalogHelpers.toDefaultConfiguredCatalog(catalog); + } + + public static ConfiguredAirbyteCatalog getConfiguredCatalogWithV1DataTypes() { + final AirbyteCatalog catalog = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + "models", + "models_schema", + io.airbyte.protocol.models.Field.of("id", JsonSchemaType.NUMBER_V1), + io.airbyte.protocol.models.Field.of("make_id", JsonSchemaType.NUMBER_V1), + io.airbyte.protocol.models.Field.of("model", JsonSchemaType.STRING_V1)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + return CatalogHelpers.toDefaultConfiguredCatalog(catalog); + } + + public static List standardSyncStates() { + final StandardSyncState standardSyncState1 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_1) + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPECIFICATION))); + final StandardSyncState standardSyncState2 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_2) + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPECIFICATION))); + final StandardSyncState standardSyncState3 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_3) + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPECIFICATION))); + final StandardSyncState standardSyncState4 = new StandardSyncState() + .withConnectionId(CONNECTION_ID_4) + .withState(new State().withState(Jsons.jsonNode(CONNECTION_SPECIFICATION))); + return Arrays.asList(standardSyncState1, standardSyncState2, standardSyncState3, standardSyncState4); + } + + public static List actorCatalogs() { + final ActorCatalog actorCatalog1 = new ActorCatalog() + .withId(ACTOR_CATALOG_ID_1) + .withCatalog(Jsons.deserialize("{}")) + .withCatalogHash("TESTHASH"); + final ActorCatalog actorCatalog2 = new ActorCatalog() + .withId(ACTOR_CATALOG_ID_2) + .withCatalog(Jsons.deserialize("{}")) + .withCatalogHash("12345"); + final ActorCatalog actorCatalog3 = new ActorCatalog() + .withId(ACTOR_CATALOG_ID_3) + .withCatalog(Jsons.deserialize("{}")) + .withCatalogHash("SomeOtherHash"); + return Arrays.asList(actorCatalog1, actorCatalog2, actorCatalog3); + } + + public static List actorCatalogFetchEvents() { + final ActorCatalogFetchEvent actorCatalogFetchEvent1 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_1) + .withActorCatalogId(ACTOR_CATALOG_ID_1) + .withActorId(SOURCE_ID_1) + .withConfigHash("CONFIG_HASH") + .withConnectorVersion("1.0.0"); + final ActorCatalogFetchEvent actorCatalogFetchEvent2 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_2) + .withActorCatalogId(ACTOR_CATALOG_ID_2) + .withActorId(SOURCE_ID_2) + .withConfigHash("1395") + .withConnectorVersion("1.42.0"); + return Arrays.asList(actorCatalogFetchEvent1, actorCatalogFetchEvent2); + } + + public static List actorCatalogFetchEventsSameSource() { + final ActorCatalogFetchEvent actorCatalogFetchEvent1 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_1) + .withActorCatalogId(ACTOR_CATALOG_ID_1) + .withActorId(SOURCE_ID_1) + .withConfigHash("CONFIG_HASH") + .withConnectorVersion("1.0.0"); + final ActorCatalogFetchEvent actorCatalogFetchEvent2 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_2) + .withActorCatalogId(ACTOR_CATALOG_ID_2) + .withActorId(SOURCE_ID_1) + .withConfigHash(CONFIG_HASH) + .withConnectorVersion(CONNECTOR_VERSION); + return Arrays.asList(actorCatalogFetchEvent1, actorCatalogFetchEvent2); + } + + @Data + public static class ActorCatalogFetchEventWithCreationDate { + + private final ActorCatalogFetchEvent actorCatalogFetchEvent; + private final OffsetDateTime createdAt; + + } + + public static List actorCatalogFetchEventsForAggregationTest() { + final OffsetDateTime now = OffsetDateTime.now(); + final OffsetDateTime yesterday = OffsetDateTime.now().minusDays(1L); + + final ActorCatalogFetchEvent actorCatalogFetchEvent1 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_1) + .withActorCatalogId(ACTOR_CATALOG_ID_1) + .withActorId(SOURCE_ID_1) + .withConfigHash("CONFIG_HASH") + .withConnectorVersion("1.0.0"); + final ActorCatalogFetchEvent actorCatalogFetchEvent2 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_2) + .withActorCatalogId(ACTOR_CATALOG_ID_2) + .withActorId(SOURCE_ID_2) + .withConfigHash(CONFIG_HASH) + .withConnectorVersion(CONNECTOR_VERSION); + final ActorCatalogFetchEvent actorCatalogFetchEvent3 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_3) + .withActorCatalogId(ACTOR_CATALOG_ID_3) + .withActorId(SOURCE_ID_2) + .withConfigHash(CONFIG_HASH) + .withConnectorVersion(CONNECTOR_VERSION); + final ActorCatalogFetchEvent actorCatalogFetchEvent4 = new ActorCatalogFetchEvent() + .withId(ACTOR_CATALOG_FETCH_EVENT_ID_3) + .withActorCatalogId(ACTOR_CATALOG_ID_3) + .withActorId(SOURCE_ID_3) + .withConfigHash(CONFIG_HASH) + .withConnectorVersion(CONNECTOR_VERSION); + return Arrays.asList( + new ActorCatalogFetchEventWithCreationDate(actorCatalogFetchEvent1, now), + new ActorCatalogFetchEventWithCreationDate(actorCatalogFetchEvent2, yesterday), + new ActorCatalogFetchEventWithCreationDate(actorCatalogFetchEvent3, now), + new ActorCatalogFetchEventWithCreationDate(actorCatalogFetchEvent4, now)); + } + + public static List workspaceServiceAccounts() { + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(WORKSPACE_ID_1) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(MOCK_SERVICE_ACCOUNT_1)); + + return Collections.singletonList(workspaceServiceAccount); + } + + public static DeclarativeManifest declarativeManifest() { + try { + return new DeclarativeManifest() + .withActorDefinitionId(UUID.randomUUID()) + .withVersion(0L) + .withDescription("a description") + .withManifest(new ObjectMapper().readTree("{\"manifest\": \"manifest\"}")) + .withSpec(new ObjectMapper().readTree("{\"spec\": \"spec\"}")); + } catch (final JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static ActorDefinitionConfigInjection actorDefinitionConfigInjection() { + try { + return new ActorDefinitionConfigInjection() + .withActorDefinitionId(UUID.randomUUID()) + .withJsonToInject(new ObjectMapper().readTree("{\"json_to_inject\": \"a json value\"}")) + .withInjectionPath("an_injection_path"); + } catch (final JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static ActiveDeclarativeManifest activeDeclarativeManifest() { + return new ActiveDeclarativeManifest().withActorDefinitionId(UUID.randomUUID()).withVersion(1L); + } + + private static Map sortMap(final Map originalMap) { + return originalMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)); + } + + public static Instant now() { + return NOW; + } + +} diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/StreamResetPersistenceTest.java b/airbyte-data/src/test/java/io/airbyte/data/services/StreamResetPersistenceTest.java new file mode 100644 index 00000000000..0be4d6b9449 --- /dev/null +++ b/airbyte-data/src/test/java/io/airbyte/data/services/StreamResetPersistenceTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.spy; + +import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class StreamResetPersistenceTest extends BaseConfigDatabaseTest { + + static StreamResetPersistence streamResetPersistence; + private static final Logger LOGGER = LoggerFactory.getLogger(StreamResetPersistenceTest.class); + + @BeforeEach + public void setup() throws Exception { + truncateAllTables(); + + streamResetPersistence = spy(new StreamResetPersistence(database)); + } + + @Test + void testCreateSameResetTwiceOnlyCreateItOnce() throws Exception { + final UUID connectionId = UUID.randomUUID(); + final StreamDescriptor streamDescriptor1 = new StreamDescriptor().withName("n1").withNamespace("ns2"); + final StreamDescriptor streamDescriptor2 = new StreamDescriptor().withName("n2"); + + streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor1, streamDescriptor2)); + + final List result = streamResetPersistence.getStreamResets(connectionId); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); + assertEquals(2, result.size()); + + streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor1)); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); + assertEquals(2, streamResetPersistence.getStreamResets(connectionId).size()); + + streamResetPersistence.createStreamResets(connectionId, List.of(streamDescriptor2)); + LOGGER.info(database.query(ctx -> ctx.selectFrom("stream_reset").fetch().toString())); + assertEquals(2, streamResetPersistence.getStreamResets(connectionId).size()); + } + + @Test + void testCreateAndGetAndDeleteStreamResets() throws Exception { + final List streamResetList = new ArrayList<>(); + final StreamDescriptor streamDescriptor1 = new StreamDescriptor().withName("stream_name_1").withNamespace("stream_namespace_1"); + final StreamDescriptor streamDescriptor2 = new StreamDescriptor().withName("stream_name_2"); + streamResetList.add(streamDescriptor1); + streamResetList.add(streamDescriptor2); + final UUID uuid = UUID.randomUUID(); + streamResetPersistence.createStreamResets(uuid, streamResetList); + + final List result = streamResetPersistence.getStreamResets(uuid); + assertEquals(2, result.size()); + assertTrue( + result.stream().anyMatch( + streamDescriptor -> "stream_name_1".equals(streamDescriptor.getName()) && "stream_namespace_1".equals(streamDescriptor.getNamespace()))); + assertTrue( + result.stream().anyMatch(streamDescriptor -> "stream_name_2".equals(streamDescriptor.getName()) && streamDescriptor.getNamespace() == null)); + + streamResetPersistence.createStreamResets(uuid, List.of(new StreamDescriptor().withName("stream_name_3").withNamespace("stream_namespace_2"))); + streamResetPersistence.deleteStreamResets(uuid, result); + + final List resultAfterDeleting = streamResetPersistence.getStreamResets(uuid); + assertEquals(1, resultAfterDeleting.size()); + + assertTrue( + resultAfterDeleting.stream().anyMatch( + streamDescriptor -> "stream_name_3".equals(streamDescriptor.getName()) && "stream_namespace_2".equals(streamDescriptor.getNamespace()))); + } + +} diff --git a/airbyte-data/src/test/java/io/airbyte/data/services/WorkspaceServiceJooqTest.java b/airbyte-data/src/test/java/io/airbyte/data/services/WorkspaceServiceJooqTest.java new file mode 100644 index 00000000000..bfca1a0bc62 --- /dev/null +++ b/airbyte-data/src/test/java/io/airbyte/data/services/WorkspaceServiceJooqTest.java @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.data.services; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.lang.MoreBooleans; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Geography; +import io.airbyte.config.Organization; +import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.ReleaseStage; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; +import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.PermissionPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; +import io.airbyte.data.services.impls.jooq.WorkspaceServiceJooqImpl; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +@SuppressWarnings({"PMD.LongVariable", "PMD.AvoidInstantiatingObjectsInLoops"}) +class WorkspaceServiceJooqTest extends BaseConfigDatabaseTest { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID SOURCE_DEFINITION_ID = UUID.randomUUID(); + private static final UUID SOURCE_ID = UUID.randomUUID(); + private static final UUID DESTINATION_DEFINITION_ID = UUID.randomUUID(); + private static final UUID DESTINATION_ID = UUID.randomUUID(); + private static final JsonNode CONFIG = Jsons.jsonNode(ImmutableMap.of("key-a", "value-a")); + + private WorkspaceServiceJooqImpl workspaceService; + private WorkspacePersistence workspacePersistence; + private PermissionPersistence permissionPersistence; + private UserPersistence userPersistence; + private ConfigRepository configRepository; + + @BeforeEach + void setup() throws Exception { + configRepository = spy(new ConfigRepository(database, null, MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); + workspacePersistence = new WorkspacePersistence(database); + permissionPersistence = new PermissionPersistence(database); + userPersistence = new UserPersistence(database); + workspaceService = spy(new WorkspaceServiceJooqImpl(database)); + final OrganizationPersistence organizationPersistence = new OrganizationPersistence(database); + + truncateAllTables(); + for (final Organization organization : MockData.organizations()) { + organizationPersistence.createOrganization(organization); + } + } + + @Test + void testGetWorkspace() throws ConfigNotFoundException, IOException, JsonValidationException { + workspaceService.writeStandardWorkspaceNoSecrets(createBaseStandardWorkspace().withWorkspaceId(UUID.randomUUID())); + assertReturnsWorkspace(createBaseStandardWorkspace()); + } + + @Test + void testWorkspaceWithNullTombstone() throws ConfigNotFoundException, IOException, JsonValidationException { + assertReturnsWorkspace(createBaseStandardWorkspace()); + } + + @Test + void testWorkspaceWithFalseTombstone() throws ConfigNotFoundException, IOException, JsonValidationException { + assertReturnsWorkspace(createBaseStandardWorkspace().withTombstone(false)); + } + + @Test + void testWorkspaceWithTrueTombstone() throws ConfigNotFoundException, IOException, JsonValidationException { + assertReturnsWorkspace(createBaseStandardWorkspace().withTombstone(true)); + } + + private static StandardWorkspace createBaseStandardWorkspace() { + return new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withName("workspace-a") + .withSlug("workspace-a-slug") + .withInitialSetupComplete(false) + .withTombstone(false) + .withDefaultGeography(Geography.AUTO); + } + + private static SourceConnection createBaseSource() { + return new SourceConnection() + .withSourceId(SOURCE_ID) + .withSourceDefinitionId(SOURCE_DEFINITION_ID) + .withName("source-a") + .withTombstone(false) + .withConfiguration(CONFIG) + .withWorkspaceId(WORKSPACE_ID); + } + + private static DestinationConnection createBaseDestination() { + return new DestinationConnection() + .withDestinationId(DESTINATION_ID) + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID) + .withName("destination-a") + .withTombstone(false) + .withConfiguration(CONFIG) + .withWorkspaceId(WORKSPACE_ID); + } + + private static StandardSourceDefinition createSourceDefinition() { + return new StandardSourceDefinition() + .withSourceDefinitionId(SOURCE_DEFINITION_ID) + .withTombstone(false) + .withName("source-definition-a"); + } + + private static ActorDefinitionVersion createActorDefinitionVersion(final UUID actorDefinitionId, + final io.airbyte.config.ReleaseStage releaseStage) { + return new ActorDefinitionVersion() + .withActorDefinitionId(actorDefinitionId) + .withDockerRepository("dockerhub") + .withSupportLevel(SupportLevel.COMMUNITY) + .withDockerImageTag("0.0.1") + .withReleaseStage(releaseStage); + } + + private static StandardDestinationDefinition createDestinationDefinition() { + return new StandardDestinationDefinition() + .withDestinationDefinitionId(DESTINATION_DEFINITION_ID) + .withTombstone(false) + .withName("destination-definition-a"); + } + + void assertReturnsWorkspace(final StandardWorkspace workspace) throws ConfigNotFoundException, IOException, JsonValidationException { + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + + final StandardWorkspace expectedWorkspace = Jsons.clone(workspace); + /* + * tombstone defaults to false in the db, so if the passed in workspace does not have it set, we + * expected the workspace returned from the db to have it set to false. + */ + if (workspace.getTombstone() == null) { + expectedWorkspace.withTombstone(false); + } + + assertEquals(workspace, workspaceService.getStandardWorkspaceNoSecrets(WORKSPACE_ID, true)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testWorkspaceByConnectionId(final boolean isTombstone) throws ConfigNotFoundException, IOException, JsonValidationException { + final UUID connectionId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); + final StandardSync mSync = new StandardSync() + .withSourceId(sourceId); + final SourceConnection mSourceConnection = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID); + final StandardWorkspace mWorkflow = new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID); + + doReturn(mSync) + .when(workspaceService) + .getStandardSync(connectionId); + doReturn(mSourceConnection) + .when(workspaceService) + .getSourceConnection(sourceId); + doReturn(mWorkflow) + .when(workspaceService) + .getStandardWorkspaceNoSecrets(WORKSPACE_ID, isTombstone); + + workspaceService.getStandardWorkspaceFromConnection(connectionId, isTombstone); + + verify(workspaceService).getStandardWorkspaceNoSecrets(WORKSPACE_ID, isTombstone); + } + + @Test + void testUpdateFeedback() throws JsonValidationException, ConfigNotFoundException, IOException { + final StandardWorkspace workspace = createBaseStandardWorkspace(); + + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + + assertFalse(MoreBooleans.isTruthy(workspaceService.getStandardWorkspaceNoSecrets(workspace.getWorkspaceId(), false).getFeedbackDone())); + workspaceService.setFeedback(workspace.getWorkspaceId()); + assertTrue(workspaceService.getStandardWorkspaceNoSecrets(workspace.getWorkspaceId(), false).getFeedbackDone()); + } + + @ParameterizedTest + @CsvSource({ + "GENERALLY_AVAILABLE, GENERALLY_AVAILABLE, false", + "ALPHA, GENERALLY_AVAILABLE, true", + "GENERALLY_AVAILABLE, BETA, true", + "CUSTOM, CUSTOM, false", + }) + void testWorkspaceHasAlphaOrBetaConnector(final ReleaseStage sourceReleaseStage, + final ReleaseStage destinationReleaseStage, + final boolean expectation) + throws JsonValidationException, IOException { + final StandardWorkspace workspace = createBaseStandardWorkspace(); + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + + configRepository.writeConnectorMetadata( + createSourceDefinition(), + createActorDefinitionVersion(SOURCE_DEFINITION_ID, sourceReleaseStage)); + configRepository.writeConnectorMetadata( + createDestinationDefinition(), + createActorDefinitionVersion(DESTINATION_DEFINITION_ID, destinationReleaseStage)); + + configRepository.writeSourceConnectionNoSecrets(createBaseSource()); + configRepository.writeDestinationConnectionNoSecrets(createBaseDestination()); + + assertEquals(expectation, workspaceService.getWorkspaceHasAlphaOrBetaConnector(WORKSPACE_ID)); + } + + @Test + void testListWorkspacesInOrgNoKeyword() throws Exception { + + final StandardWorkspace workspace = createBaseStandardWorkspace().withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1); + final StandardWorkspace otherWorkspace = createBaseStandardWorkspace().withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_2); + + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + workspaceService.writeStandardWorkspaceNoSecrets(otherWorkspace); + + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( + new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.empty()); + assertReturnsWorkspace(createBaseStandardWorkspace().withTombstone(false)); + + assertEquals(1, workspaces.size()); + assertEquals(workspace, workspaces.get(0)); + } + + @Test + void testListWorkspacesInOrgWithPagination() throws Exception { + final StandardWorkspace workspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("A workspace"); + final StandardWorkspace otherWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()).withOrganizationId(MockData.ORGANIZATION_ID_1).withName("B workspace"); + + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + workspaceService.writeStandardWorkspaceNoSecrets(otherWorkspace); + + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( + new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 1, 0), Optional.empty()); + + assertEquals(1, workspaces.size()); + assertEquals(workspace, workspaces.get(0)); + } + + @Test + void testListWorkspacesInOrgWithKeyword() throws Exception { + final StandardWorkspace workspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspaceWithKeyword"); + final StandardWorkspace otherWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()).withOrganizationId(MockData.ORGANIZATION_ID_1).withName("workspace"); + + workspaceService.writeStandardWorkspaceNoSecrets(workspace); + workspaceService.writeStandardWorkspaceNoSecrets(otherWorkspace); + + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( + new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.of("keyword")); + + assertEquals(1, workspaces.size()); + assertEquals(workspace, workspaces.get(0)); + } + + @Test + void testGetDefaultWorkspaceForOrganization() throws JsonValidationException, IOException { + final StandardWorkspace expectedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspaceInOrganization1"); + + workspaceService.writeStandardWorkspaceNoSecrets(expectedWorkspace); + + final StandardWorkspace tombstonedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("tombstonedWorkspace") + .withTombstone(true); + + workspaceService.writeStandardWorkspaceNoSecrets(tombstonedWorkspace); + + final StandardWorkspace laterWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("laterWorkspace"); + + workspaceService.writeStandardWorkspaceNoSecrets(laterWorkspace); + + final StandardWorkspace actualWorkspace = workspacePersistence.getDefaultWorkspaceForOrganization(MockData.ORGANIZATION_ID_1); + + assertEquals(expectedWorkspace, actualWorkspace); + } + + @Test + void testListWorkspacesByUserIdWithKeywordWithPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1, name contains search "keyword" + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace_with_keyword_1"); + workspaceService.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2, name contains search "Keyword" + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace_with_Keyword_2"); + workspaceService.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserIdPaginated( + new ResourcesByUserQueryPaginated(userId, false, 10, 0), Optional.of("keyWord")); + + assertEquals(2, workspaces.size()); + } + + @Test + void testListWorkspacesByUserIdWithoutKeywordWithoutPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1 + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace1"); + workspaceService.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2 + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace2"); + workspaceService.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserId( + userId, false, Optional.empty()); + + assertEquals(2, workspaces.size()); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/DatabaseConstants.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/DatabaseConstants.java index 41b62e613e5..03e8a2094a2 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/DatabaseConstants.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/DatabaseConstants.java @@ -69,6 +69,14 @@ public final class DatabaseConstants { public static final String WORKSPACE_TABLE = "workspace"; + public static final String ORGANIZATION_TABLE = "organization"; + + public static final String USER_INVITATION_TABLE = "user_invitation"; + + public static final String ORGANIZATION_EMAIL_DOMAIN_TABLE = "organization_email_domain"; + + public static final String SSO_CONFIG_TABLE = "sso_config"; + /** * Private constructor to prevent instantiation. */ diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java new file mode 100644 index 00000000000..b3cced62a6b --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import com.google.common.annotations.VisibleForTesting; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Sets all actor's default_version_id to its actor_definition's default_version_id, and sets the + * column to be non-null. + */ +public class V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.class); + + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final Table ACTOR = DSL.table("actor"); + + private static final Field ID = DSL.field("id", SQLDataType.UUID); + private static final Field DEFAULT_VERSION_ID = DSL.field("default_version_id", SQLDataType.UUID); + private static final Field ACTOR_DEFINITION_ID = DSL.field("actor_definition_id", SQLDataType.UUID); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + backfillActorDefaultVersionId(ctx); + setNonNull(ctx); + } + + @VisibleForTesting + static void backfillActorDefaultVersionId(final DSLContext ctx) { + final var actorDefinitions = ctx.select(ID, DEFAULT_VERSION_ID) + .from(ACTOR_DEFINITION) + .fetch(); + + for (final var actorDefinition : actorDefinitions) { + final UUID actorDefinitionId = actorDefinition.get(ID); + final UUID defaultVersionId = actorDefinition.get(DEFAULT_VERSION_ID); + + ctx.update(ACTOR) + .set(DEFAULT_VERSION_ID, defaultVersionId) + .where(ACTOR_DEFINITION_ID.eq(actorDefinitionId)) + .execute(); + } + } + + @VisibleForTesting + static void setNonNull(final DSLContext ctx) { + ctx.alterTable(ACTOR) + .alterColumn(DEFAULT_VERSION_ID) + .setNotNull() + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java new file mode 100644 index 00000000000..97586d012b5 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import com.google.common.annotations.VisibleForTesting; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Changes the message column type of actor_definition_breaking_change to CLOB, which will set\ it + * to 'text' in the db. We want to be able to handle large messages. + */ +public class V0_50_23_002__SetBreakingChangesMessageColumnToClobType extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_23_002__SetBreakingChangesMessageColumnToClobType.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + alterMessageColumnType(ctx); + } + + @VisibleForTesting + static void alterMessageColumnType(final DSLContext ctx) { + ctx.alterTable("actor_definition_breaking_change") + .alter("message").set(SQLDataType.CLOB).execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java new file mode 100644 index 00000000000..8a1f4baa45d --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Inserts a support_level column to the actor_definition_version table. + */ +public class V0_50_23_003__AddSupportLevelToActorDefinitionVersion extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_23_003__AddSupportLevelToActorDefinitionVersion.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + final DSLContext ctx = DSL.using(context.getConnection()); + addSupportLevelToActorDefinitionVersion(ctx); + LOGGER.info("support_level column added to actor_definition_version table"); + } + + static void addSupportLevelToActorDefinitionVersion(final DSLContext ctx) { + createSupportLevelEnum(ctx); + addSupportLevelColumn(ctx); + } + + public static void createSupportLevelEnum(final DSLContext ctx) { + ctx.createType("support_level").asEnum("community", "certified", "none").execute(); + } + + static void addSupportLevelColumn(final DSLContext ctx) { + ctx.alterTable("actor_definition_version") + .addColumnIfNotExists(DSL.field("support_level", SQLDataType.VARCHAR.asEnumDataType( + V0_50_23_003__AddSupportLevelToActorDefinitionVersion.SupportLevel.class).nullable(false).defaultValue(SupportLevel.none))) + .execute(); + } + + enum SupportLevel implements EnumType { + + community("community"), + certified("certified"), + none("none"); + + private final String literal; + + SupportLevel(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "support_level"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java new file mode 100644 index 00000000000..14e86ccaa0e --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a migration to naively populate all actor_definition_version records with a support_level + * relative to their release stage. alpha -> community beta -> community general_availability -> + * certified + */ +public class V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger( + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.class); + + static void backfillSupportLevel(final DSLContext ctx) { + ctx.transaction(configuration -> { + final var transactionCtx = DSL.using(configuration); + + final var updateQuery = + "UPDATE actor_definition_version SET support_level = {0} WHERE release_stage = {1} AND support_level = 'none'::support_level"; + + // For all connections with invalid catalog, update to valid catalog + transactionCtx.execute(updateQuery, SupportLevel.community, ReleaseStage.alpha); + transactionCtx.execute(updateQuery, SupportLevel.community, ReleaseStage.beta); + transactionCtx.execute(updateQuery, SupportLevel.certified, ReleaseStage.generally_available); + transactionCtx.execute(updateQuery, SupportLevel.none, ReleaseStage.custom); + + // Drop the default Support Level + transactionCtx.alterTable("actor_definition_version").alterColumn("support_level").dropDefault().execute(); + }); + } + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + + backfillSupportLevel(ctx); + } + + enum SupportLevel implements EnumType { + + community("community"), + certified("certified"), + none("none"); + + private final String literal; + + SupportLevel(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "support_level"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + enum ReleaseStage implements EnumType { + + alpha("alpha"), + beta("beta"), + generally_available("generally_available"), + custom("custom"); + + private final String literal; + + ReleaseStage(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "release_stage"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_24_001__Add_UserInvitation_OrganizationEmailDomain_SsoConfig_Tables.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_24_001__Add_UserInvitation_OrganizationEmailDomain_SsoConfig_Tables.java new file mode 100644 index 00000000000..0b38103fbe2 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_24_001__Add_UserInvitation_OrganizationEmailDomain_SsoConfig_Tables.java @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static io.airbyte.db.instance.DatabaseConstants.ORGANIZATION_EMAIL_DOMAIN_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.ORGANIZATION_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.SSO_CONFIG_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.USER_INVITATION_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.USER_TABLE; +import static io.airbyte.db.instance.DatabaseConstants.WORKSPACE_TABLE; +import static org.jooq.impl.DSL.currentOffsetDateTime; +import static org.jooq.impl.DSL.foreignKey; +import static org.jooq.impl.DSL.primaryKey; +import static org.jooq.impl.DSL.unique; + +import io.airbyte.db.instance.configs.migrations.V0_50_16_001__UpdateEnumTypeAuthProviderAndPermissionType.PermissionType; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Field; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Adding new tables and corresponding indexes: UserInvitation, OrganizationEmailDomain, and + * SsoConfig. + */ +public class V0_50_24_001__Add_UserInvitation_OrganizationEmailDomain_SsoConfig_Tables extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_24_001__Add_UserInvitation_OrganizationEmailDomain_SsoConfig_Tables.class); + + private static final String ORGANIZATION_ID = "organization_id"; + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + + createInvitationStatusEnumType(ctx); + createUserInvitationTableAndIndexes(ctx); + createOrganizationEmailDomainTableAndIndexes(ctx); + createSsoConfigTableAndIndexes(ctx); + } + + private static void createInvitationStatusEnumType(final DSLContext ctx) { + ctx.createType(InvitationStatus.NAME) + .asEnum(Arrays.stream(InvitationStatus.values()).map(InvitationStatus::getLiteral).toList()) + .execute(); + } + + private static void createUserInvitationTableAndIndexes(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field inviteCode = DSL.field("invite_code", SQLDataType.VARCHAR(256).nullable(false)); + final Field inviterUserId = DSL.field("inviter_user_id", SQLDataType.UUID.nullable(false)); + final Field invitedEmail = DSL.field("invited_email", SQLDataType.VARCHAR(256).nullable(false)); + final Field workspaceId = DSL.field("workspace_id", SQLDataType.UUID.nullable(true)); + final Field organizationId = DSL.field(ORGANIZATION_ID, SQLDataType.UUID.nullable(true)); + final Field permissionType = + DSL.field("permission_type", SQLDataType.VARCHAR.asEnumDataType(PermissionType.class).nullable(false)); + final Field status = + DSL.field("status", SQLDataType.VARCHAR.asEnumDataType(InvitationStatus.class).nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists(USER_INVITATION_TABLE) + .columns( + id, + inviteCode, + inviterUserId, + invitedEmail, + workspaceId, + organizationId, + permissionType, + status, + createdAt, + updatedAt) + .constraints( + primaryKey(id), + // an invite code needs to be able to uniquely identify an invitation. + unique(inviteCode), + // don't remove an invitation if the inviter is deleted. + foreignKey(inviterUserId).references(USER_TABLE, "id").onDeleteNoAction(), + // if a workspace is deleted, remove any invitation to that workspace. + foreignKey(workspaceId).references(WORKSPACE_TABLE, "id").onDeleteCascade(), + // if an organization is deleted, remove any invitation to that organization. + foreignKey(organizationId).references(ORGANIZATION_TABLE, "id").onDeleteCascade()) + .execute(); + + // create an index on invite_code, for efficient lookups of a particular invitation by invite code. + ctx.createIndexIfNotExists("user_invitation_invite_code_idx") + .on(USER_INVITATION_TABLE, "invite_code") + .execute(); + + // create an index on invited email, for efficient lookups of all invitations sent to a particular + // email address. + ctx.createIndexIfNotExists("user_invitation_invited_email_idx") + .on(USER_INVITATION_TABLE, "invited_email") + .execute(); + + // create an index on workspace id, for efficient lookups of all invitations to a particular + // workspace. + ctx.createIndexIfNotExists("user_invitation_workspace_id_idx") + .on(USER_INVITATION_TABLE, "workspace_id") + .execute(); + + // create an index on organization id, for efficient lookups of all invitations to a particular + // organization. + ctx.createIndexIfNotExists("user_invitation_organization_id_idx") + .on(USER_INVITATION_TABLE, ORGANIZATION_ID) + .execute(); + } + + private static void createOrganizationEmailDomainTableAndIndexes(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field organizationId = DSL.field(ORGANIZATION_ID, SQLDataType.UUID.nullable(false)); + final Field emailDomain = DSL.field("email_domain", SQLDataType.VARCHAR(256).nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists(ORGANIZATION_EMAIL_DOMAIN_TABLE) + .columns( + id, + organizationId, + emailDomain, + createdAt) + .constraints( + primaryKey(id), + // an email domain can only be associated with one organization. + unique(emailDomain), + // if an organization is deleted, remove any email domains associated with that organization. + foreignKey(organizationId).references(ORGANIZATION_TABLE, "id").onDeleteCascade()) + .execute(); + + // create an index on domain, for efficient lookups of a particular email domain. + ctx.createIndexIfNotExists("organization_email_domain_email_domain_idx") + .on("organization_email_domain", "email_domain") + .execute(); + + // create an index on organization id, for efficient lookups of all email domains associated with a + // particular organization. + ctx.createIndexIfNotExists("organization_email_domain_organization_id_idx") + .on("organization_email_domain", ORGANIZATION_ID) + .execute(); + } + + private static void createSsoConfigTableAndIndexes(final DSLContext ctx) { + // create table with id, organization_id, and keycloak_realm fields + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field organizationId = DSL.field(ORGANIZATION_ID, SQLDataType.UUID.nullable(false)); + final Field keycloakRealm = DSL.field("keycloak_realm", SQLDataType.VARCHAR(256).nullable(false)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists(SSO_CONFIG_TABLE) + .columns( + id, + organizationId, + keycloakRealm, + createdAt, + updatedAt) + .constraints( + primaryKey(id), + // an organization can only have one SSO config. + unique(organizationId), + // a keycloak realm can only be associated with one organization. + unique(keycloakRealm), + // if an organization is deleted, remove any SSO config associated with that organization. + foreignKey(organizationId).references(ORGANIZATION_TABLE, "id").onDeleteCascade()) + .execute(); + + // create an index on organization id, for efficient lookups of a particular organization's SSO + // config. + ctx.createIndexIfNotExists("sso_config_organization_id_idx") + .on(SSO_CONFIG_TABLE, ORGANIZATION_ID) + .execute(); + + // create an index on keycloak realm, for efficient lookups of a particular SSO config by keycloak + // realm. + ctx.createIndexIfNotExists("sso_config_keycloak_realm_idx") + .on(SSO_CONFIG_TABLE, "keycloak_realm") + .execute(); + } + + /** + * new InvitationStatus enum. + */ + public enum InvitationStatus implements EnumType { + + PENDING("pending"), + ACCEPTED("accepted"), + CANCELLED("cancelled"); + + private final String literal; + public static final String NAME = "invitation_status"; + + InvitationStatus(final String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public")); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index b8e7055617a..62df592ed10 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -20,7 +20,7 @@ create table "public"."actor" ( "tombstone" boolean not null default false, "created_at" timestamp(6) with time zone not null default current_timestamp, "updated_at" timestamp(6) with time zone not null default current_timestamp, - "default_version_id" uuid, + "default_version_id" uuid not null, constraint "actor_pkey" primary key ("id") ); @@ -66,7 +66,7 @@ create table "public"."actor_definition_breaking_change" ( "version" varchar(256) not null, "migration_documentation_url" varchar(256) not null, "upgrade_deadline" date not null, - "message" varchar(256) not null, + "message" text not null, "created_at" timestamp(6) with time zone not null default current_timestamp, "updated_at" timestamp(6) with time zone not null default current_timestamp, constraint "actor_definition_breaking_change_pkey" @@ -100,6 +100,7 @@ create table "public"."actor_definition_version" ( "suggested_streams" jsonb, "release_stage" "public"."release_stage", "support_state" "public"."support_state" not null default cast('supported' as support_state), + "support_level" "public"."support_level" not null, constraint "actor_definition_version_pkey" primary key ("id") ); @@ -226,6 +227,16 @@ create table "public"."organization" ( constraint "organization_pkey" primary key ("id") ); +create table "public"."organization_email_domain" ( + "id" uuid not null, + "organization_id" uuid not null, + "email_domain" varchar(256) not null, + "created_at" timestamp(6) with time zone not null default current_timestamp, + constraint "organization_email_domain_pkey" + primary key ("id"), + constraint "organization_email_domain_email_domain_key" + unique ("email_domain") +); create table "public"."permission" ( "id" uuid not null, "user_id" uuid not null, @@ -246,6 +257,19 @@ create table "public"."schema_management" ( constraint "schema_management_pkey" primary key ("id") ); +create table "public"."sso_config" ( + "id" uuid not null, + "organization_id" uuid not null, + "keycloak_realm" varchar(256) not null, + "created_at" timestamp(6) with time zone not null default current_timestamp, + "updated_at" timestamp(6) with time zone not null default current_timestamp, + constraint "sso_config_pkey" + primary key ("id"), + constraint "sso_config_keycloak_realm_key" + unique ("keycloak_realm"), + constraint "sso_config_organization_id_key" + unique ("organization_id") +); create table "public"."state" ( "id" uuid not null, "connection_id" uuid not null, @@ -286,6 +310,22 @@ create table "public"."user" ( constraint "user_pkey" primary key ("id") ); +create table "public"."user_invitation" ( + "id" uuid not null, + "invite_code" varchar(256) not null, + "inviter_user_id" uuid not null, + "invited_email" varchar(256) not null, + "workspace_id" uuid, + "organization_id" uuid, + "permission_type" "public"."permission_type" not null, + "status" "public"."invitation_status" not null, + "created_at" timestamp(6) with time zone not null default current_timestamp, + "updated_at" timestamp(6) with time zone not null default current_timestamp, + constraint "user_invitation_pkey" + primary key ("id"), + constraint "user_invitation_invite_code_key" + unique ("invite_code") +); create table "public"."workspace" ( "id" uuid not null, "customer_id" uuid, @@ -393,6 +433,10 @@ alter table "public"."operation" add constraint "operation_workspace_id_fkey" foreign key ("workspace_id") references "public"."workspace" ("id"); +alter table "public"."organization_email_domain" + add constraint "organization_email_domain_organization_id_fkey" + foreign key ("organization_id") + references "public"."organization" ("id"); alter table "public"."permission" add constraint "permission_organization_id_fkey" foreign key ("organization_id") @@ -409,6 +453,10 @@ alter table "public"."schema_management" add constraint "schema_management_connection_id_fkey" foreign key ("connection_id") references "public"."connection" ("id"); +alter table "public"."sso_config" + add constraint "sso_config_organization_id_fkey" + foreign key ("organization_id") + references "public"."organization" ("id"); alter table "public"."state" add constraint "state_connection_id_fkey" foreign key ("connection_id") @@ -417,6 +465,18 @@ alter table "public"."user" add constraint "user_default_workspace_id_fkey" foreign key ("default_workspace_id") references "public"."workspace" ("id"); +alter table "public"."user_invitation" + add constraint "user_invitation_inviter_user_id_fkey" + foreign key ("inviter_user_id") + references "public"."user" ("id"); +alter table "public"."user_invitation" + add constraint "user_invitation_organization_id_fkey" + foreign key ("organization_id") + references "public"."organization" ("id"); +alter table "public"."user_invitation" + add constraint "user_invitation_workspace_id_fkey" + foreign key ("workspace_id") + references "public"."workspace" ("id"); alter table "public"."workspace" add constraint "workspace_organization_id_fkey" foreign key ("organization_id") @@ -439,10 +499,18 @@ create index "connection_source_id_idx" on "public"."connection"("source_id" asc create index "connection_status_idx" on "public"."connection"("status" asc); create index "connection_operation_connection_id_idx" on "public"."connection_operation"("connection_id" asc); create index "connector_builder_project_workspace_idx" on "public"."connector_builder_project"("workspace_id" asc); +create index "organization_email_domain_email_domain_idx" on "public"."organization_email_domain"("email_domain" asc); +create index "organization_email_domain_organization_id_idx" on "public"."organization_email_domain"("organization_id" asc); create index "permission_organization_id_idx" on "public"."permission"("organization_id" asc); create index "permission_user_id_idx" on "public"."permission"("user_id" asc); create index "permission_workspace_id_idx" on "public"."permission"("workspace_id" asc); create index "connection_idx" on "public"."schema_management"("connection_id" asc); +create index "sso_config_keycloak_realm_idx" on "public"."sso_config"("keycloak_realm" asc); +create index "sso_config_organization_id_idx" on "public"."sso_config"("organization_id" asc); create index "connection_id_stream_name_namespace_idx" on "public"."stream_reset"("connection_id" asc, "stream_name" asc, "stream_namespace" asc); create index "user_auth_provider_auth_user_id_idx" on "public"."user"("auth_provider" asc, "auth_user_id" asc); create index "user_email_idx" on "public"."user"("email" asc); +create index "user_invitation_invite_code_idx" on "public"."user_invitation"("invite_code" asc); +create index "user_invitation_invited_email_idx" on "public"."user_invitation"("invited_email" asc); +create index "user_invitation_organization_id_idx" on "public"."user_invitation"("organization_id" asc); +create index "user_invitation_workspace_id_idx" on "public"."user_invitation"("workspace_id" asc); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java new file mode 100644 index 00000000000..b151845c13f --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.util.Objects; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest extends AbstractConfigsDatabaseTest { + + private static final Table ACTOR = DSL.table("actor"); + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final Table ACTOR_DEFINITION_VERSION = DSL.table("actor_definition_version"); + private static final Table WORKSPACE = DSL.table("workspace"); + + private static final Field ID_COL = DSL.field("id", SQLDataType.UUID); + private static final Field DEFAULT_VERSION_ID_COL = DSL.field("default_version_id", SQLDataType.UUID); + private static final Field ACTOR_DEFINITION_ID_COL = DSL.field("actor_definition_id", SQLDataType.UUID); + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + private static final UUID ACTOR_ID = UUID.randomUUID(); + private static final UUID VERSION_ID = UUID.randomUUID(); + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_20_001__MakeManualNullableForRemoval(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + private UUID getDefaultVersionIdForActorId(final DSLContext ctx, final UUID actorId) { + final var actor = ctx.select(DEFAULT_VERSION_ID_COL) + .from(ACTOR) + .where(ID_COL.eq(actorId)) + .fetchOne(); + + if (Objects.isNull(actor)) { + return null; + } + + return actor.get(DEFAULT_VERSION_ID_COL); + } + + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + static void insertDependencies(final DSLContext ctx) { + ctx.insertInto(WORKSPACE) + .columns( + ID_COL, + DSL.field("name"), + DSL.field("slug"), + DSL.field("initial_setup_complete")) + .values( + WORKSPACE_ID, + "name1", + "default", + true) + .execute(); + + ctx.insertInto(ACTOR_DEFINITION) + .columns( + ID_COL, + DSL.field("name"), + DSL.field("actor_type")) + .values( + ACTOR_DEFINITION_ID, + "source def name", + ActorType.source) + .execute(); + + ctx.insertInto(ACTOR_DEFINITION_VERSION) + .columns(ID_COL, ACTOR_DEFINITION_ID_COL, DSL.field("docker_repository"), DSL.field("docker_image_tag"), DSL.field("spec")) + .values(VERSION_ID, ACTOR_DEFINITION_ID, "airbyte/some-source", "1.0.0", JSONB.valueOf("{}")) + .execute(); + + ctx.update(ACTOR_DEFINITION) + .set(DEFAULT_VERSION_ID_COL, VERSION_ID) + .where(ID_COL.eq(ACTOR_DEFINITION_ID)) + .execute(); + } + + @Test + void testBackFillActorDefaultVersionId() { + final DSLContext ctx = getDslContext(); + insertDependencies(ctx); + + ctx.insertInto(ACTOR) + .columns( + ID_COL, + ACTOR_DEFINITION_ID_COL, + DSL.field("workspace_id"), + DSL.field("name"), + DSL.field("configuration"), + DSL.field("actor_type")) + .values( + ACTOR_ID, + ACTOR_DEFINITION_ID, + WORKSPACE_ID, + "My Source", + JSONB.valueOf("{}"), + ActorType.source) + .execute(); + + assertNull(getDefaultVersionIdForActorId(ctx, ACTOR_ID)); + + V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.backfillActorDefaultVersionId(ctx); + + assertEquals(VERSION_ID, getDefaultVersionIdForActorId(ctx, ACTOR_ID)); + } + + @Test + void testActorDefaultVersionIdIsNotNull() { + final DSLContext context = getDslContext(); + + V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.setNonNull(context); + + final Exception e = Assertions.assertThrows(DataAccessException.class, () -> { + context.insertInto(ACTOR) + .columns( + ID_COL, + ACTOR_DEFINITION_ID_COL, + DSL.field("workspace_id"), + DSL.field("name"), + DSL.field("configuration"), + DSL.field("actor_type")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + "My Source", + JSONB.valueOf("{}"), + ActorType.source) + .execute(); + }); + Assertions.assertTrue(e.getMessage().contains("null value in column \"default_version_id\" of relation \"actor\" violates not-null constraint")); + } + +} diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java new file mode 100644 index 00000000000..dd73e44beb4 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static io.airbyte.db.instance.configs.migrations.V0_50_23_002__SetBreakingChangesMessageColumnToClobType.alterMessageColumnType; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest extends AbstractConfigsDatabaseTest { + + private static final Table ACTOR_DEFINITION_BREAKING_CHANGE = DSL.table("actor_definition_breaking_change"); + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + @Test + void testInsertThrowsBeforeMigration() { + final DSLContext ctx = getDslContext(); + insertActorDefinitionDependency(ctx); + final Throwable exception = assertThrows(DataAccessException.class, () -> insertBreakingChange(ctx)); + assertTrue(exception.getMessage().contains("value too long for type character varying(256)")); + } + + @Test + void testInsertSucceedsAfterMigration() { + final DSLContext ctx = getDslContext(); + insertActorDefinitionDependency(ctx); + alterMessageColumnType(ctx); + assertDoesNotThrow(() -> insertBreakingChange(ctx)); + } + + private static void insertActorDefinitionDependency(final DSLContext ctx) { + ctx.insertInto(ACTOR_DEFINITION) + .columns( + DSL.field("id"), + DSL.field("name"), + DSL.field("actor_type")) + .values( + ACTOR_DEFINITION_ID, + "source def name", + ActorType.source) + .onConflict( + DSL.field("id")) + .doNothing() + .execute(); + } + + private static void insertBreakingChange(final DSLContext ctx) { + final String message = + "This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early."; + + ctx.insertInto(ACTOR_DEFINITION_BREAKING_CHANGE) + .columns( + DSL.field("actor_definition_id"), + DSL.field("version"), + DSL.field("upgrade_deadline"), + DSL.field("message"), + DSL.field("migration_documentation_url"), + DSL.field("created_at"), + DSL.field("updated_at")) + .values( + ACTOR_DEFINITION_ID, + "3.0.0", + Date.valueOf("2023-11-01"), + message, + "https://docs.airbyte.com/integrations/destinations/snowflake-migrations#3.0.0", + Timestamp.valueOf("2023-08-25 16:33:42.701943875"), + Timestamp.valueOf("2023-08-25 16:33:42.701943875")) + .onConflict( + DSL.field("actor_definition_id"), + DSL.field("version")) + .doUpdate() + .set(DSL.field("upgrade_deadline"), Date.valueOf("2023-11-01")) + .set(DSL.field("message"), message) + .set(DSL.field("migration_documentation_url"), "https://docs.airbyte.com/integrations/destinations/snowflake-migrations#3.0.0") + .set(DSL.field("updated_at"), Timestamp.valueOf("2023-08-25 16:33:42.701943875")) + .execute(); + + } + +} diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java new file mode 100644 index 00000000000..1cc6bde0508 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_50_23_003__AddSupportLevelToActorDefinitionVersion.SupportLevel; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.io.IOException; +import java.sql.SQLException; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest extends AbstractConfigsDatabaseTest { + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_6_002__AddDefaultVersionIdToActor(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + @Test + void test() throws SQLException, IOException { + final DSLContext context = getDslContext(); + + // ignore all foreign key constraints + context.execute("SET session_replication_role = replica;"); + + Assertions.assertFalse(supportLevelColumnExists(context)); + + V0_50_23_003__AddSupportLevelToActorDefinitionVersion.addSupportLevelToActorDefinitionVersion(context); + + Assertions.assertTrue(supportLevelColumnExists(context)); + + assertSupportLevelEnumWorks(context); + } + + private static boolean supportLevelColumnExists(final DSLContext ctx) { + return ctx.fetchExists(DSL.select() + .from("information_schema.columns") + .where(DSL.field("table_name").eq("actor_definition_version") + .and(DSL.field("column_name").eq("support_level")))); + } + + private static void assertSupportLevelEnumWorks(final DSLContext ctx) { + Assertions.assertDoesNotThrow(() -> { + insertWithSupportLevel(ctx, SupportLevel.community); + insertWithSupportLevel(ctx, SupportLevel.certified); + insertWithSupportLevel(ctx, SupportLevel.none); + }); + + Assertions.assertThrows(Exception.class, () -> { + insertWithSupportLevel(ctx, SupportLevel.valueOf("invalid")); + }); + + Assertions.assertThrows(Exception.class, () -> { + insertWithSupportLevel(ctx, SupportLevel.valueOf(null)); + }); + } + + private static void insertWithSupportLevel(final DSLContext ctx, final SupportLevel supportLevel) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("support_level")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + supportLevel) + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java new file mode 100644 index 00000000000..30f42f9f397 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.ReleaseStage; +import io.airbyte.db.instance.configs.migrations.V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.SupportLevel; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.util.List; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +class V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest extends AbstractConfigsDatabaseTest { + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + private static void insertAdvWithReleaseStageAndSupportLevel( + final DSLContext ctx, + final ReleaseStage releaseStage, + final SupportLevel supportLevel) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("support_level"), + DSL.field("release_stage")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + supportLevel, + releaseStage) + .execute(); + } + + private static void insertWithoutSupportLevel(final DSLContext ctx) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("release_stage")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + ReleaseStage.alpha) + .execute(); + } + + @Test + void testBackfillSupportLevel() throws Exception { + final DSLContext ctx = getDslContext(); + + // ignore all foreign key constraints + ctx.execute("SET session_replication_role = replica;"); + + final int numberOfAdvs = 10; + + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.alpha, SupportLevel.certified); + + for (int i = 0; i < numberOfAdvs; i++) { + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.alpha, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.beta, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.generally_available, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.custom, SupportLevel.none); + } + + // assert that all advs have support level "none" + final List preAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("support_level").eq(SupportLevel.none)) + .fetch(); + + Assertions.assertEquals(numberOfAdvs * 4, preAdvs.size()); + + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.backfillSupportLevel(ctx); + + // assert that all alpha advs have support level set to community + final List alphaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.alpha).and(DSL.field("support_level").eq(SupportLevel.community))) + .fetch(); + Assertions.assertEquals(numberOfAdvs, alphaAdvs.size()); + + // assert that all beta advs have support level set to community + final List betaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.beta).and(DSL.field("support_level").eq(SupportLevel.community))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, betaAdvs.size()); + + // assert that all generally_available advs have support level set to certified + final List gaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.generally_available).and(DSL.field("support_level").eq(SupportLevel.certified))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, gaAdvs.size()); + + // assert that all custom advs have support level set to none + final List customAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.custom).and(DSL.field("support_level").eq(SupportLevel.none))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, customAdvs.size()); + + // assert that there is one adv with support level certified and release stage alpha (i.e. did not + // get overwritten) + final List certifiedAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.alpha).and(DSL.field("support_level").eq(SupportLevel.certified))) + .fetch(); + + Assertions.assertEquals(1, certifiedAdvs.size()); + } + + @Test + void testNoDefaultSupportLevel() { + final DSLContext ctx = getDslContext(); + + // ignore all foreign key constraints + ctx.execute("SET session_replication_role = replica;"); + + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.backfillSupportLevel(ctx); + + Assertions.assertThrows(RuntimeException.class, () -> insertWithoutSupportLevel(ctx)); + } + +} diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index fb46158d4aa..bd46be4f19b 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -41,9 +41,11 @@ object AutoPropagateNewStreams : Temporary(key = "autopropagate-new-str object CanonicalCatalogSchema : Temporary(key = "canonical-catalog-schema", default = false) -object CheckConnectionUseApiEnabled : Temporary(key = "check-connection-use-api", default = false) +object CatalogCanonicalJson : Temporary(key = "catalog-canonical-json", default = false) -object CheckConnectionUseChildWorkflowEnabled : Temporary(key = "check-connection-use-child-workflow", default = false) +object EarlySyncEnabled : Temporary(key = "billing.early-sync-enabled", default = false) + +object FetchEarlySyncJobs : Temporary(key = "billing.fetch-early-sync-jobs", default = false) object ShouldRunOnGkeDataplane : Temporary(key = "should-run-on-gke-dataplane", default = false) @@ -61,9 +63,9 @@ object ShouldFailSyncIfHeartbeatFailure : Permanent(key = "heartbeat.fa object ConnectorVersionOverride : Permanent(key = "connectors.versionOverrides", default = "") -object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = false) +object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = true) -object IngestBreakingChanges : Temporary(key = "connectors.ingestBreakingChanges", default = true) +object RunSupportStateUpdater : Temporary(key = "connectors.runSupportStateUpdater", default = true) object RefreshSchemaPeriod : Temporary(key = "refreshSchema.period.hours", default = 24) @@ -99,7 +101,15 @@ object UseCustomK8sScheduler : Temporary(key = "platform.use-custom-k8s- object HideActorDefinitionFromList : Permanent(key = "connectors.hideActorDefinitionFromList", default = false) -object UseNewStateMessageProcessing : Temporary(key = "platform.use-new-state-message-processing", default = false) +object PauseSyncsWithUnsupportedActors : Temporary(key = "connectors.pauseSyncsWithUnsupportedActors", default = true) + +object SunsetFCP : Temporary(key = "platform.sunset-fcp", default = false) + +object DestResourceOverrides : Temporary(key = "dest-resource-overrides", default = "") + +object OrchestratorResourceOverrides : Temporary(key = "orchestrator-resource-overrides", default = "") + +object SourceResourceOverrides : Temporary(key = "source-resource-overrides", default = "") // NOTE: this is deprecated in favor of FieldSelectionEnabled and will be removed once that flag is fully deployed. object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") { diff --git a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java index 7c42cd9742f..7eb6495c107 100644 --- a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java +++ b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java @@ -42,7 +42,8 @@ public IdentityProvidersCreator(final List identi public void createIdps(final RealmResource keycloakRealm) { // Create Identity Providers if (identityProviderConfigurations == null || identityProviderConfigurations.isEmpty()) { - throw new RuntimeException("No providers found in config"); + log.info("No identity providers configured. Skipping IDP setup."); + return; } for (final IdentityProviderConfiguration provider : identityProviderConfigurations) { diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java index b938fae66b5..7ce444e875b 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java @@ -79,6 +79,11 @@ public static final class Tags { */ public static final String DOCKER_IMAGE_KEY = "docker_image"; + /** + * Name of the APM trace tag that holds the actual type of the error. + */ + public static final String ERROR_ACTUAL_TYPE_KEY = "error.actual_type"; + /** * Name of the APM trace tag that holds the failure origin(s) associated with the trace. */ diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java index 0bf2fccecb6..e8547961edb 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java @@ -4,6 +4,7 @@ package io.airbyte.metrics.lib; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; @@ -77,12 +78,15 @@ public static void addTagsToTrace(final Span span, final Map tag * All tags added via this method will use the default {@link #TAG_PREFIX} namespace. Any null * values will be ignored. */ - public static void addTagsToTrace(final UUID connectionId, final String jobId, final Path jobRoot) { + public static void addTagsToTrace(final UUID connectionId, final Long attemptNumber, final String jobId, final Path jobRoot) { final Map tags = new HashMap<>(); if (connectionId != null) { tags.put(CONNECTION_ID_KEY, connectionId); } + if (attemptNumber != null) { + tags.put(ATTEMPT_NUMBER_KEY, attemptNumber); + } if (jobId != null) { tags.put(JOB_ID_KEY, jobId); } diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java index b6610c50a93..5b97ccf303c 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java @@ -82,7 +82,7 @@ public void count(final MetricsRegistry metric, final long amt, final MetricAttr return; } - log.info("publishing count, name: {}, value: {}, attributes: {}", metric, amt, attributes); + log.debug("publishing count, name: {}, value: {}, attributes: {}", metric, amt, attributes); statsDClient.count(metric.getMetricName(), amt, toTags(attributes)); } } diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 7b346f8f4c3..f7d8c0261e1 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -214,6 +214,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { REPLICATION_WORKER_CREATED(MetricEmittingApps.WORKER, "replication_worker_created", "number of replication worker created"), + REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR(MetricEmittingApps.WORKER, + "replication_worker_executor_shutdown_error", + "number of failure to shutdown executors"), REPLICATION_MADE_PROGRESS(MetricEmittingApps.WORKER, "replication_made_progress", "Count of replication runs that made progress. To be faceted by attributes."), @@ -270,6 +273,12 @@ public enum OssMetricsRegistry implements MetricsRegistry { "state_timestamp_metric_tracker_error", "number of syncs where the state timestamp metric tracker ran out of memory or " + "was unable to match destination state message to source state message"), + STATE_PROCESSED_FROM_DESTINATION(MetricEmittingApps.WORKER, + "state_processed_from_destination", + "counter for number of state messages received from destination"), + STATE_PROCESSED_FROM_SOURCE(MetricEmittingApps.WORKER, + "state_processed_from_source", + "counter for number of state messages received from source"), // TEMPORARY, delete after the migration. STATS_TRACKER_IMPLEMENTATION(MetricEmittingApps.WORKER, "stats_tracker_implementation", @@ -277,6 +286,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { STREAM_STATS_WRITE_NUM_QUERIES(MetricEmittingApps.WORKER, "stream_stats_write_num_queries", "number of separate queries to update the stream stats table"), + TEMPORAL_API_TRANSIENT_ERROR_RETRY(MetricEmittingApps.WORKER, + "temporal_api_transient_error_retry", + "whenever we retry a temporal api call for transient errors"), TEMPORAL_WORKFLOW_ATTEMPT(MetricEmittingApps.WORKER, "temporal_workflow_attempt", "count of the number of workflow attempts"), @@ -308,6 +320,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { WORKER_SOURCE_MESSAGE_READ(MetricEmittingApps.WORKER, "worker_source_message_read", "whenever a message is read from the source"), + WORFLOW_UNREACHABLE(MetricEmittingApps.WORKER, + "workflow_unreachable", + "whenever a workflow is unreachable"), WORKFLOWS_HEALED(MetricEmittingApps.CRON, "workflows_healed", "number of workflow the self healing cron healed"), diff --git a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java index 06c1ceef021..e927ab2dedf 100644 --- a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java +++ b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java @@ -4,6 +4,7 @@ package io.airbyte.metrics.lib; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; @@ -134,27 +135,39 @@ void testAddingTagsWithNullChecks() { final UUID connectionID = UUID.randomUUID(); final String jobId = UUID.randomUUID().toString(); final Path jobRoot = Path.of("dev", "null"); + final Long attemptNumber = Long.valueOf(2L); - ApmTraceUtils.addTagsToTrace(connectionID, jobId, jobRoot); + ApmTraceUtils.addTagsToTrace(connectionID, attemptNumber, jobId, jobRoot); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace(null, jobId, jobRoot); + ApmTraceUtils.addTagsToTrace(null, null, jobId, jobRoot); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace(connectionID, jobId, null); + ApmTraceUtils.addTagsToTrace(connectionID, null, jobId, null); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace((UUID) null, null, null); + ApmTraceUtils.addTagsToTrace(null, attemptNumber, jobId, null); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), jobRoot.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); + + clearInvocations(span); + ApmTraceUtils.addTagsToTrace((UUID) null, null, null, null); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); } diff --git a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java index 3078db34019..86d22a759f1 100644 --- a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java +++ b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java @@ -11,6 +11,7 @@ import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_CATALOG_FETCH_EVENT; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE; import static io.airbyte.db.instance.jobs.jooq.generated.Tables.ATTEMPTS; @@ -24,6 +25,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType; import io.airbyte.db.instance.configs.jooq.generated.enums.NamespaceDefinitionType; import io.airbyte.db.instance.configs.jooq.generated.enums.StatusType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.jobs.jooq.generated.enums.AttemptStatus; import io.airbyte.db.instance.jobs.jooq.generated.enums.JobConfigType; import io.airbyte.db.instance.jobs.jooq.generated.enums.JobStatus; @@ -58,6 +60,8 @@ class MetricRepositoryTest { private static final UUID SRC_DEF_ID = UUID.randomUUID(); private static final UUID DST_DEF_ID = UUID.randomUUID(); + private static final UUID SRC_DEF_VER_ID = UUID.randomUUID(); + private static final UUID DST_DEF_VER_ID = UUID.randomUUID(); private static MetricRepository db; private static DSLContext ctx; @@ -80,6 +84,13 @@ public static void setUpAll() throws DatabaseInitializationException, IOExceptio .values(UUID.randomUUID(), "dstDef", ActorType.destination) .execute(); + ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC, + ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .execute(); + // drop constraints to simplify test set up ctx.alterTable(ACTOR).dropForeignKey(ACTOR__ACTOR_WORKSPACE_ID_FKEY.constraint()).execute(); ctx.alterTable(CONNECTION).dropForeignKey(CONNECTION__CONNECTION_DESTINATION_ID_FKEY.constraint()).execute(); @@ -318,10 +329,11 @@ void shouldReturnNumConnectionsBasic() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, @@ -345,10 +357,11 @@ void shouldIgnoreNonRunningConnections() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, @@ -374,10 +387,11 @@ void shouldIgnoreDeletedWorkspaces() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index d2fbaa5da75..5c36f3b50ac 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -27,12 +27,15 @@ import io.airbyte.config.provider.ResourceRequirementsProvider; import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.Context; +import io.airbyte.featureflag.DestResourceOverrides; import io.airbyte.featureflag.Destination; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.OrchestratorResourceOverrides; import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; +import io.airbyte.featureflag.SourceResourceOverrides; import io.airbyte.featureflag.UseResourceRequirementsVariant; import io.airbyte.featureflag.Workspace; import io.airbyte.protocol.models.CatalogHelpers; @@ -174,15 +177,17 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac final StandardSourceDefinition sourceDefinition, final StandardDestinationDefinition destinationDefinition, final boolean isReset) { - final String variant = getResourceRequirementsVariant(workspaceId, standardSync, sourceDefinition, destinationDefinition); + final var ffContext = buildFeatureFlagContext(workspaceId, standardSync, sourceDefinition, destinationDefinition); + final String variant = featureFlagClient.stringVariation(UseResourceRequirementsVariant.INSTANCE, ffContext); // Note on use of sourceType, throughput is driven by the source, if the source is slow, the rest is // going to be slow. With this in mind, we align the resources given to the orchestrator and the // destination based on the source to avoid oversizing orchestrator and destination when the source // is slow. final Optional sourceType = getSourceType(sourceDefinition); - final ResourceRequirements mergedOrchestratorResourceReq = getOrchestratorResourceRequirements(standardSync, sourceType, variant); - final ResourceRequirements mergedDstResourceReq = getDestinationResourceRequirements(standardSync, destinationDefinition, sourceType, variant); + final ResourceRequirements mergedOrchestratorResourceReq = getOrchestratorResourceRequirements(standardSync, sourceType, variant, ffContext); + final ResourceRequirements mergedDstResourceReq = + getDestinationResourceRequirements(standardSync, destinationDefinition, sourceType, variant, ffContext); final var syncResourceRequirements = new SyncResourceRequirements() .withConfigKey(new SyncResourceRequirementsKey().withVariant(variant).withSubType(sourceType.orElse(null))) @@ -194,7 +199,7 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac .withOrchestrator(mergedOrchestratorResourceReq); if (!isReset) { - final ResourceRequirements mergedSrcResourceReq = getSourceResourceRequirements(standardSync, sourceDefinition, variant); + final ResourceRequirements mergedSrcResourceReq = getSourceResourceRequirements(standardSync, sourceDefinition, variant, ffContext); syncResourceRequirements .withSource(mergedSrcResourceReq) .withSourceStdErr(resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE_STDERR, sourceType, variant)) @@ -204,10 +209,10 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac return syncResourceRequirements; } - private String getResourceRequirementsVariant(final UUID workspaceId, - final StandardSync standardSync, - final StandardSourceDefinition sourceDefinition, - final StandardDestinationDefinition destinationDefinition) { + private Context buildFeatureFlagContext(final UUID workspaceId, + final StandardSync standardSync, + final StandardSourceDefinition sourceDefinition, + final StandardDestinationDefinition destinationDefinition) { final List contextList = new ArrayList<>(); addIfNotNull(contextList, workspaceId, Workspace::new); addIfNotNull(contextList, standardSync.getConnectionId(), Connection::new); @@ -216,7 +221,7 @@ private String getResourceRequirementsVariant(final UUID workspaceId, addIfNotNull(contextList, sourceDefinition != null ? sourceDefinition.getSourceDefinitionId() : null, SourceDefinition::new); addIfNotNull(contextList, standardSync.getDestinationId(), Destination::new); addIfNotNull(contextList, destinationDefinition.getDestinationDefinitionId(), DestinationDefinition::new); - return featureFlagClient.stringVariation(UseResourceRequirementsVariant.INSTANCE, new Multi(contextList)); + return new Multi(contextList); } private static void addIfNotNull(final List contextList, final UUID uuid, final Function supplier) { @@ -227,37 +232,85 @@ private static void addIfNotNull(final List contextList, final UUID uui private ResourceRequirements getOrchestratorResourceRequirements(final StandardSync standardSync, final Optional sourceType, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultOrchestratorRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.ORCHESTRATOR, sourceType, variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRrsReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), defaultOrchestratorRssReqs); + + final var overrides = getOrchestratorResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRrsReqs); } private ResourceRequirements getSourceResourceRequirements(final StandardSync standardSync, final StandardSourceDefinition sourceDefinition, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultSrcRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, getSourceType(sourceDefinition), variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRssReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), sourceDefinition != null ? sourceDefinition.getResourceRequirements() : null, defaultSrcRssReqs, JobType.SYNC); + + final var overrides = getSourceResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRssReqs); } private ResourceRequirements getDestinationResourceRequirements(final StandardSync standardSync, final StandardDestinationDefinition destinationDefinition, final Optional sourceType, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultDstRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.DESTINATION, sourceType, variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRssReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), destinationDefinition.getResourceRequirements(), defaultDstRssReqs, JobType.SYNC); + + final var overrides = getDestinationResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRssReqs); + } + + private ResourceRequirements getDestinationResourceOverrides(final Context ffCtx) { + final String destOverrides = featureFlagClient.stringVariation(DestResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(destOverrides); + } catch (final Exception e) { + log.warn("Could not parse DESTINATION resource overrides '{}' from feature flag string: {}", destOverrides, e.getMessage()); + return null; + } + } + + private ResourceRequirements getOrchestratorResourceOverrides(final Context ffCtx) { + final String orchestratorOverrides = featureFlagClient.stringVariation(OrchestratorResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(orchestratorOverrides); + } catch (final Exception e) { + log.warn("Could not parse ORCHESTRATOR resource overrides '{}' from feature flag string: {}", orchestratorOverrides, e.getMessage()); + return null; + } + } + + private ResourceRequirements getSourceResourceOverrides(final Context ffCtx) { + final String sourceOverrides = featureFlagClient.stringVariation(SourceResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(sourceOverrides); + } catch (final Exception e) { + log.warn("Could not parse SOURCE resource overrides '{}' from feature flag string: {}", sourceOverrides, e.getMessage()); + return null; + } } private Optional getSourceType(final StandardSourceDefinition sourceDefinition) { diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java new file mode 100644 index 00000000000..949eabf4cc2 --- /dev/null +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.persistence.job; + +import io.airbyte.commons.version.AirbyteProtocolVersion; +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; +import io.airbyte.db.Database; +import io.airbyte.db.ExceptionWrappingDatabase; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Encapsulates jobs db interactions for the metadata domain models. + */ +public class DefaultMetadataPersistence implements MetadataPersistence { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetadataPersistence.class); + + private static final String AIRBYTE_METADATA_TABLE = "airbyte_metadata"; + private static final String METADATA_KEY_COL = "key"; + private static final String METADATA_VAL_COL = "value"; + private static final String DEPLOYMENT_ID_KEY = "deployment_id"; + + private final ExceptionWrappingDatabase jobDatabase; + + public DefaultMetadataPersistence(final Database database) { + this.jobDatabase = new ExceptionWrappingDatabase(database); + } + + @Override + public Optional getVersion() throws IOException { + return getMetadata(AirbyteVersion.AIRBYTE_VERSION_KEY_NAME).findFirst(); + } + + @Override + public void setVersion(final String airbyteVersion) throws IOException { + // This is not using setMetadata due to the extra (s_init_db, airbyteVersion) that is + // added to the metadata table + jobDatabase.query(ctx -> ctx.execute(String.format( + "INSERT INTO %s(%s, %s) VALUES('%s', '%s'), ('%s_init_db', '%s') ON CONFLICT (%s) DO UPDATE SET %s = '%s'", + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + METADATA_VAL_COL, + AirbyteVersion.AIRBYTE_VERSION_KEY_NAME, + airbyteVersion, + ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME), + airbyteVersion, + METADATA_KEY_COL, + METADATA_VAL_COL, + airbyteVersion))); + + } + + @Override + public Optional getAirbyteProtocolVersionMax() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMax(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME, version.serialize()); + } + + @Override + public Optional getAirbyteProtocolVersionMin() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMin(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME, version.serialize()); + } + + @Override + public Optional getCurrentProtocolVersionRange() throws IOException { + final Optional min = getAirbyteProtocolVersionMin(); + final Optional max = getAirbyteProtocolVersionMax(); + + if (min.isPresent() != max.isPresent()) { + // Flagging this because this would be highly suspicious but not bad enough that we should fail + // hard. + // If the new config is fine, the system should self-heal. + LOGGER.warn("Inconsistent AirbyteProtocolVersion found, only one of min/max was found. (min:{}, max:{})", + min.map(Version::serialize).orElse(""), max.map(Version::serialize).orElse("")); + } + + if (min.isEmpty() && max.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(new AirbyteProtocolVersionRange(min.orElse(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION), + max.orElse(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION))); + } + + private Stream getMetadata(final String keyName) throws IOException { + return jobDatabase.query(ctx -> ctx.select() + .from(AIRBYTE_METADATA_TABLE) + .where(DSL.field(METADATA_KEY_COL).eq(keyName)) + .fetch()).stream().map(r -> r.getValue(METADATA_VAL_COL, String.class)); + } + + private void setMetadata(final String keyName, final String value) throws IOException { + jobDatabase.query(ctx -> ctx + .insertInto(DSL.table(AIRBYTE_METADATA_TABLE)) + .columns(DSL.field(METADATA_KEY_COL), DSL.field(METADATA_VAL_COL)) + .values(keyName, value) + .onConflict(DSL.field(METADATA_KEY_COL)) + .doUpdate() + .set(DSL.field(METADATA_VAL_COL), value) + .execute()); + } + + @Override + public Optional getDeployment() throws IOException { + final Result result = jobDatabase.query(ctx -> ctx.select() + .from(AIRBYTE_METADATA_TABLE) + .where(DSL.field(METADATA_KEY_COL).eq(DEPLOYMENT_ID_KEY)) + .fetch()); + return result.stream().findFirst().map(r -> UUID.fromString(r.getValue(METADATA_VAL_COL, String.class))); + } + + @Override + public void setDeployment(final UUID deployment) throws IOException { + // if an existing deployment id already exists, on conflict, return it so we can log it. + final UUID committedDeploymentId = jobDatabase.query(ctx -> ctx.fetch(String.format( + "INSERT INTO %s(%s, %s) VALUES('%s', '%s') ON CONFLICT (%s) DO NOTHING RETURNING (SELECT %s FROM %s WHERE %s='%s') as existing_deployment_id", + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + METADATA_VAL_COL, + DEPLOYMENT_ID_KEY, + deployment, + METADATA_KEY_COL, + METADATA_VAL_COL, + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + DEPLOYMENT_ID_KEY))) + .stream() + .filter(record -> record.get("existing_deployment_id", String.class) != null) + .map(record -> UUID.fromString(record.get("existing_deployment_id", String.class))) + .findFirst() + .orElse(deployment); // if no record was returned that means that the new deployment id was used. + + if (!deployment.equals(committedDeploymentId)) { + LOGGER.warn("Attempted to set a deployment id {}, but deployment id {} already set. Retained original value.", deployment, deployment); + } + } + +} diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java new file mode 100644 index 00000000000..6ea59d23500 --- /dev/null +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.persistence.job; + +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.Version; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +/** + * General persistence interface for storing metadata. + */ +public interface MetadataPersistence { + + /** + * Returns the AirbyteVersion. + */ + Optional getVersion() throws IOException; + + /** + * Set the airbyte version. + */ + void setVersion(String airbyteVersion) throws IOException; + + /** + * Get the max supported Airbyte Protocol Version. + */ + Optional getAirbyteProtocolVersionMax() throws IOException; + + /** + * Set the max supported Airbyte Protocol Version. + */ + void setAirbyteProtocolVersionMax(Version version) throws IOException; + + /** + * Get the min supported Airbyte Protocol Version. + */ + Optional getAirbyteProtocolVersionMin() throws IOException; + + /** + * Set the min supported Airbyte Protocol Version. + */ + void setAirbyteProtocolVersionMin(Version version) throws IOException; + + /** + * Get the current Airbyte Protocol Version range if defined. + */ + Optional getCurrentProtocolVersionRange() throws IOException; + + /** + * Returns a deployment UUID. + */ + Optional getDeployment() throws IOException; + // a deployment references a setup of airbyte. it is created the first time the docker compose or + // K8s is ready. + + /** + * Set deployment id. If one is already set, the new value is ignored. + */ + void setDeployment(UUID uuid) throws IOException; + +} diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java index 58c04e616c4..10cf5c9f0e7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java @@ -5,6 +5,7 @@ package io.airbyte.persistence.job; import com.google.common.base.Preconditions; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.JobTypeResourceLimit; import io.airbyte.config.JobTypeResourceLimit.JobType; @@ -137,4 +138,19 @@ public static Optional getResourceRequirementsForJobType(f : Optional.of(jobTypeResourceRequirement.get(0)); } + /** + * Utility for deserializing from a raw json string. + * + * @param rawOverrides A json string to be parsed. + * @return ResourceRequirements parsed from the string. + */ + public static ResourceRequirements parse(final String rawOverrides) { + if (rawOverrides.isEmpty()) { + return null; + } + + final var json = Jsons.deserialize(rawOverrides); + return Jsons.object(json, ResourceRequirements.class); + } + } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java index 70679bec540..4974e9eaa65 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java @@ -44,6 +44,8 @@ public class WorkspaceHelper { private final LoadingCache operationToWorkspaceCache; private final LoadingCache jobToWorkspaceCache; + private final LoadingCache workspaceToOrganizationCache; + public WorkspaceHelper(final ConfigRepository configRepository, final JobPersistence jobPersistence) { this.sourceToWorkspaceCache = getExpiringCache(new CacheLoader<>() { @@ -104,6 +106,15 @@ public UUID load(@NonNull final Long jobId) throws ConfigNotFoundException, IOEx } }); + + this.workspaceToOrganizationCache = getExpiringCache(new CacheLoader<>() { + + @Override + public UUID load(UUID workspaceId) throws Exception { + return configRepository.getStandardWorkspaceNoSecrets(workspaceId, false).getOrganizationId(); + } + + }); } /** @@ -141,6 +152,10 @@ public UUID getWorkspaceForJobIdIgnoreExceptions(final Long jobId) { return swallowExecutionException(() -> jobToWorkspaceCache.get(jobId)); } + public UUID getOrganizationForWorkspace(final UUID workspaceId) { + return swallowExecutionException(() -> workspaceToOrganizationCache.get(workspaceId)); + } + // CONNECTION ID /** diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java index 74d9082899b..7702d1c51a7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java @@ -142,89 +142,94 @@ private static Map generateActorDefinitionVersionMetadata(final */ public static Map generateJobAttemptMetadata(final Job job) { final Builder metadata = ImmutableMap.builder(); - if (job != null) { - final List attempts = job.getAttempts(); - if (attempts != null && !attempts.isEmpty()) { - final Attempt lastAttempt = attempts.get(attempts.size() - 1); - if (lastAttempt.getOutput() != null && lastAttempt.getOutput().isPresent()) { - final JobOutput jobOutput = lastAttempt.getOutput().get(); - if (jobOutput.getSync() != null) { - final StandardSyncSummary syncSummary = jobOutput.getSync().getStandardSyncSummary(); - final SyncStats totalStats = syncSummary.getTotalStats(); - final NormalizationSummary normalizationSummary = jobOutput.getSync().getNormalizationSummary(); - - if (syncSummary.getStartTime() != null) { - metadata.put("sync_start_time", syncSummary.getStartTime()); - } - if (syncSummary.getEndTime() != null && syncSummary.getStartTime() != null) { - metadata.put("duration", Math.round((syncSummary.getEndTime() - syncSummary.getStartTime()) / 1000.0)); - } - if (syncSummary.getBytesSynced() != null) { - metadata.put("volume_mb", syncSummary.getBytesSynced()); - } - if (syncSummary.getRecordsSynced() != null) { - metadata.put("volume_rows", syncSummary.getRecordsSynced()); - } - if (totalStats.getSourceStateMessagesEmitted() != null) { - metadata.put("count_state_messages_from_source", syncSummary.getTotalStats().getSourceStateMessagesEmitted()); - } - if (totalStats.getDestinationStateMessagesEmitted() != null) { - metadata.put("count_state_messages_from_destination", syncSummary.getTotalStats().getDestinationStateMessagesEmitted()); - } - if (totalStats.getMaxSecondsBeforeSourceStateMessageEmitted() != null) { - metadata.put("max_seconds_before_source_state_message_emitted", - totalStats.getMaxSecondsBeforeSourceStateMessageEmitted()); - } - if (totalStats.getMeanSecondsBeforeSourceStateMessageEmitted() != null) { - metadata.put("mean_seconds_before_source_state_message_emitted", - totalStats.getMeanSecondsBeforeSourceStateMessageEmitted()); - } - if (totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted() != null) { - metadata.put("max_seconds_between_state_message_emit_and_commit", - totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()); - } - if (totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted() != null) { - metadata.put("mean_seconds_between_state_message_emit_and_commit", - totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()); - } + // Early returns in case we're missing the relevant stats. + if (job == null) { + return metadata.build(); + } + final List attempts = job.getAttempts(); + if (attempts == null || attempts.isEmpty()) { + return metadata.build(); + } + final Attempt lastAttempt = attempts.get(attempts.size() - 1); + if (lastAttempt.getOutput() == null || lastAttempt.getOutput().isEmpty()) { + return metadata.build(); + } + final JobOutput jobOutput = lastAttempt.getOutput().get(); + if (jobOutput.getSync() == null) { + return metadata.build(); + } + final StandardSyncSummary syncSummary = jobOutput.getSync().getStandardSyncSummary(); + final SyncStats totalStats = syncSummary.getTotalStats(); + final NormalizationSummary normalizationSummary = jobOutput.getSync().getNormalizationSummary(); - if (totalStats.getReplicationStartTime() != null) { - metadata.put("replication_start_time", totalStats.getReplicationStartTime()); - } - if (totalStats.getReplicationEndTime() != null) { - metadata.put("replication_end_time", totalStats.getReplicationEndTime()); - } - if (totalStats.getSourceReadStartTime() != null) { - metadata.put("source_read_start_time", totalStats.getSourceReadStartTime()); - } - if (totalStats.getSourceReadEndTime() != null) { - metadata.put("source_read_end_time", totalStats.getSourceReadEndTime()); - } - if (totalStats.getDestinationWriteStartTime() != null) { - metadata.put("destination_write_start_time", totalStats.getDestinationWriteStartTime()); - } - if (totalStats.getDestinationWriteEndTime() != null) { - metadata.put("destination_write_end_time", totalStats.getDestinationWriteEndTime()); - } + if (syncSummary.getStartTime() != null) { + metadata.put("sync_start_time", syncSummary.getStartTime()); + } + if (syncSummary.getEndTime() != null && syncSummary.getStartTime() != null) { + metadata.put("duration", Math.round((syncSummary.getEndTime() - syncSummary.getStartTime()) / 1000.0)); + } + if (syncSummary.getBytesSynced() != null) { + metadata.put("volume_mb", syncSummary.getBytesSynced()); + } + if (syncSummary.getRecordsSynced() != null) { + metadata.put("volume_rows", syncSummary.getRecordsSynced()); + } + if (totalStats.getSourceStateMessagesEmitted() != null) { + metadata.put("count_state_messages_from_source", syncSummary.getTotalStats().getSourceStateMessagesEmitted()); + } + if (totalStats.getDestinationStateMessagesEmitted() != null) { + metadata.put("count_state_messages_from_destination", syncSummary.getTotalStats().getDestinationStateMessagesEmitted()); + } + if (totalStats.getMaxSecondsBeforeSourceStateMessageEmitted() != null) { + metadata.put("max_seconds_before_source_state_message_emitted", + totalStats.getMaxSecondsBeforeSourceStateMessageEmitted()); + } + if (totalStats.getMeanSecondsBeforeSourceStateMessageEmitted() != null) { + metadata.put("mean_seconds_before_source_state_message_emitted", + totalStats.getMeanSecondsBeforeSourceStateMessageEmitted()); + } + if (totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted() != null) { + metadata.put("max_seconds_between_state_message_emit_and_commit", + totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()); + } + if (totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted() != null) { + metadata.put("mean_seconds_between_state_message_emit_and_commit", + totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()); + } - if (normalizationSummary != null) { - if (normalizationSummary.getStartTime() != null) { - metadata.put("normalization_start_time", normalizationSummary.getStartTime()); + if (totalStats.getReplicationStartTime() != null) { + metadata.put("replication_start_time", totalStats.getReplicationStartTime()); + } + if (totalStats.getReplicationEndTime() != null) { + metadata.put("replication_end_time", totalStats.getReplicationEndTime()); + } + if (totalStats.getSourceReadStartTime() != null) { + metadata.put("source_read_start_time", totalStats.getSourceReadStartTime()); + } + if (totalStats.getSourceReadEndTime() != null) { + metadata.put("source_read_end_time", totalStats.getSourceReadEndTime()); + } + if (totalStats.getDestinationWriteStartTime() != null) { + metadata.put("destination_write_start_time", totalStats.getDestinationWriteStartTime()); + } + if (totalStats.getDestinationWriteEndTime() != null) { + metadata.put("destination_write_end_time", totalStats.getDestinationWriteEndTime()); + } - } - if (normalizationSummary.getEndTime() != null) { - metadata.put("normalization_end_time", normalizationSummary.getEndTime()); - } - } - } - } + if (normalizationSummary != null) { + if (normalizationSummary.getStartTime() != null) { + metadata.put("normalization_start_time", normalizationSummary.getStartTime()); - final List failureReasons = failureReasonsList(attempts); - if (!failureReasons.isEmpty()) { - metadata.put("failure_reasons", failureReasonsListAsJson(failureReasons).toString()); - metadata.put("main_failure_reason", failureReasonAsJson(failureReasons.get(0)).toString()); - } } + if (normalizationSummary.getEndTime() != null) { + metadata.put("normalization_end_time", normalizationSummary.getEndTime()); + } + } + + final List failureReasons = failureReasonsList(attempts); + if (!failureReasons.isEmpty()) { + metadata.put("failure_reasons", failureReasonsListAsJson(failureReasons).toString()); + metadata.put("main_failure_reason", failureReasonAsJson(failureReasons.get(0)).toString()); } return metadata.build(); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index 2c2f419902f..745f88f6244 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -41,6 +41,9 @@ import io.airbyte.config.SyncResourceRequirements; import io.airbyte.config.SyncResourceRequirementsKey; import io.airbyte.config.provider.ResourceRequirementsProvider; +import io.airbyte.featureflag.DestResourceOverrides; +import io.airbyte.featureflag.OrchestratorResourceOverrides; +import io.airbyte.featureflag.SourceResourceOverrides; import io.airbyte.featureflag.TestClient; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -51,12 +54,21 @@ import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.protocol.models.SyncMode; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.commons.util.StringUtils; +import org.mockito.ArgumentCaptor; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") class DefaultJobCreatorTest { private static final String DEFAULT_VARIANT = "default"; @@ -82,6 +94,7 @@ class DefaultJobCreatorTest { private static final StandardDestinationDefinition STANDARD_DESTINATION_DEFINITION; private static final ActorDefinitionVersion SOURCE_DEFINITION_VERSION; private static final ActorDefinitionVersion DESTINATION_DEFINITION_VERSION; + private static final ConfiguredAirbyteCatalog CONFIGURED_AIRBYTE_CATALOG; private static final long JOB_ID = 12L; private static final UUID WORKSPACE_ID = UUID.randomUUID(); @@ -141,7 +154,7 @@ class DefaultJobCreatorTest { .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) .withSyncMode(SyncMode.FULL_REFRESH) .withDestinationSyncMode(DestinationSyncMode.OVERWRITE); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream1, stream2, stream3)); + CONFIGURED_AIRBYTE_CATALOG = new ConfiguredAirbyteCatalog().withStreams(List.of(stream1, stream2, stream3)); STANDARD_SYNC = new StandardSync() .withConnectionId(connectionId) @@ -150,7 +163,7 @@ class DefaultJobCreatorTest { .withNamespaceFormat(null) .withPrefix("presto_to_hudi") .withStatus(StandardSync.Status.ACTIVE) - .withCatalog(catalog) + .withCatalog(CONFIGURED_AIRBYTE_CATALOG) .withSourceId(sourceId) .withDestinationId(destinationId) .withOperationIds(List.of(operationId)); @@ -507,6 +520,271 @@ void testCreateSyncJobSourceAndDestinationResourceReqs() throws IOException { verify(jobPersistence, times(1)).enqueueJob(expectedScope, expectedJobConfig); } + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testDestinationResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(DestResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var destConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getDestination(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, destConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, destConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, destConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, destConfigValues.getMemoryLimit()); + } + + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testOrchestratorResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(OrchestratorResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + final var standardSync = new StandardSync() + .withConnectionId(UUID.randomUUID()) + .withName("presto to hudi") + .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) + .withNamespaceFormat(null) + .withPrefix("presto_to_hudi") + .withStatus(StandardSync.Status.ACTIVE) + .withCatalog(CONFIGURED_AIRBYTE_CATALOG) + .withSourceId(UUID.randomUUID()) + .withDestinationId(UUID.randomUUID()) + .withOperationIds(List.of(UUID.randomUUID())) + .withResourceRequirements(originalReqs); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + standardSync, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(destResourceRequirements)), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var orchestratorConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getOrchestrator(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, orchestratorConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, orchestratorConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, orchestratorConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, orchestratorConfigValues.getMemoryLimit()); + } + + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testSourceResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(SourceResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(destResourceRequirements)), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var sourceConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getSource(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, sourceConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, sourceConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, sourceConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, sourceConfigValues.getMemoryLimit()); + } + + private static Stream resourceOverrideMatrix() { + return Stream.of( + Arguments.of("0.7", "0.4", "1000Mi", "2000Mi"), + Arguments.of("0.3", null, "1000Mi", null), + Arguments.of(null, null, null, null), + Arguments.of(null, "0.4", null, null), + Arguments.of("3", "3", "3000Mi", "3000Mi"), + Arguments.of("4", "5", "6000Mi", "7000Mi")); + } + + @ParameterizedTest + @MethodSource("weirdnessOverrideMatrix") + void ignoresOverridesIfJsonStringWeird(final String weirdness) throws IOException { + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(DestResourceOverrides.INSTANCE.getKey(), Jsons.serialize(weirdness)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var destConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getDestination(); + + assertEquals(originalReqs.getCpuRequest(), destConfigValues.getCpuRequest()); + assertEquals(originalReqs.getCpuLimit(), destConfigValues.getCpuLimit()); + assertEquals(originalReqs.getMemoryRequest(), destConfigValues.getMemoryRequest()); + assertEquals(originalReqs.getMemoryLimit(), destConfigValues.getMemoryLimit()); + } + + private static Stream weirdnessOverrideMatrix() { + return Stream.of( + Arguments.of("0.7"), + Arguments.of("0.5, 1, 1000Mi, 2000Mi"), + Arguments.of("cat burglar"), + Arguments.of("{ \"cpu_limit\": \"2\", \"cpu_request\": \"1\" "), + Arguments.of("null"), + Arguments.of("undefined"), + Arguments.of(""), + Arguments.of("{}")); + } + @Test void testCreateResetConnectionJob() throws IOException { final Optional expectedSourceType = Optional.empty(); diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java index b7e44136d4a..73f4a32e5ad 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java @@ -4,11 +4,13 @@ package io.airbyte.persistence.job.tracker; +import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import io.airbyte.config.AttemptSyncConfig; +import io.airbyte.config.JobConfig; import io.airbyte.config.JobOutput; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSync; @@ -18,15 +20,19 @@ import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptStatus; import io.airbyte.persistence.job.models.Job; +import io.airbyte.persistence.job.models.JobStatus; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.Test; class TrackingMetadataTest { + private static final String UNUSED = "unused"; + @Test void testNulls() { final UUID connectionId = UUID.randomUUID(); @@ -79,4 +85,37 @@ void testgenerateJobAttemptMetadataWithNulls() { assertEquals(expected, actual); } + @Test + void testGenerateJobAttemptMetadataToleratesNullInputs() { + // Null job. + assertTrue(TrackingMetadata.generateJobAttemptMetadata(null).isEmpty()); + + // There is a job, but it has no attempts. + final Job jobWithNoAttempts = new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNoAttempts).isEmpty()); + + // There is a job, and it has an attempt, but the attempt has null output. + final Attempt mockAttemptWithNullOutput = mock(Attempt.class); + when(mockAttemptWithNullOutput.getOutput()).thenReturn(null); + final Job jobWithNullOutput = + new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNullOutput).isEmpty()); + + // There is a job, and it has an attempt, but the attempt has empty output. + final Attempt mockAttemptWithEmptyOutput = mock(Attempt.class); + when(mockAttemptWithEmptyOutput.getOutput()).thenReturn(Optional.empty()); + final Job jobWithEmptyOutput = + new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithEmptyOutput).isEmpty()); + + // There is a job, and it has an attempt, and the attempt has output, but the output has no sync + // info. + final Attempt mockAttemptWithOutput = mock(Attempt.class); + final JobOutput mockJobOutputWithoutSync = mock(JobOutput.class); + when(mockAttemptWithOutput.getOutput()).thenReturn(Optional.of(mockJobOutputWithoutSync)); + when(mockJobOutputWithoutSync.getSync()).thenReturn(null); + final Job jobWithoutSyncInfo = new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithoutSyncInfo).isEmpty()); + } + } diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index 177ea6521e2..77d7a247215 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:latest -ARG VERSION=0.50.23 +ARG VERSION=dev ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 6d684fde080..1661a8a71c2 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 -ARG VERSION=0.50.23 +ARG VERSION=dev ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 55eae2fbc30..ed6fd241185 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -65,7 +65,6 @@ dependencies { implementation libs.bundles.datadog implementation libs.sentry.java implementation libs.swagger.annotations - implementation libs.javax.ws.rs.api implementation libs.google.cloud.storage runtimeOnly libs.javax.databind diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java index 4f899471809..e8e19969b7f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/AttemptApiController.java @@ -10,6 +10,8 @@ import io.airbyte.api.generated.AttemptApi; import io.airbyte.api.model.generated.AttemptInfoRead; import io.airbyte.api.model.generated.AttemptStats; +import io.airbyte.api.model.generated.CreateNewAttemptNumberRequest; +import io.airbyte.api.model.generated.CreateNewAttemptNumberResponse; import io.airbyte.api.model.generated.GetAttemptStatsRequestBody; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.SaveAttemptSyncConfigRequestBody; @@ -48,6 +50,17 @@ public AttemptInfoRead getAttemptForJob(final GetAttemptStatsRequestBody request .execute(() -> attemptHandler.getAttemptForJob(requestBody.getJobId(), requestBody.getAttemptNumber())); } + @Override + @Post(uri = "/create_new_attempt_number", + processes = MediaType.APPLICATION_JSON) + @ExecuteOn(AirbyteTaskExecutors.IO) + @Secured({ADMIN}) + @SecuredWorkspace + public CreateNewAttemptNumberResponse createNewAttemptNumber(CreateNewAttemptNumberRequest createNewAttemptNumberRequest) { + return ApiHelper + .execute(() -> attemptHandler.createNewAttemptNumber(createNewAttemptNumberRequest.getJobId())); + } + @Override @Post(uri = "/get_combined_stats", processes = MediaType.APPLICATION_JSON) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java index 2f43aa3733c..068fab087d9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiController.java @@ -10,7 +10,7 @@ import io.airbyte.api.model.generated.DestinationDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.DestinationDefinitionSpecificationRead; import io.airbyte.api.model.generated.DestinationIdRequestBody; -import io.airbyte.commons.server.handlers.SchedulerHandler; +import io.airbyte.commons.server.handlers.ConnectorDefinitionSpecificationHandler; import io.airbyte.commons.server.scheduling.AirbyteTaskExecutors; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -23,10 +23,10 @@ @Secured(SecurityRule.IS_AUTHENTICATED) public class DestinationDefinitionSpecificationApiController implements DestinationDefinitionSpecificationApi { - private final SchedulerHandler schedulerHandler; + private final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler; - public DestinationDefinitionSpecificationApiController(final SchedulerHandler schedulerHandler) { - this.schedulerHandler = schedulerHandler; + public DestinationDefinitionSpecificationApiController(final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler) { + this.connectorDefinitionSpecificationHandler = connectorDefinitionSpecificationHandler; } @SuppressWarnings("LineLength") @@ -35,7 +35,7 @@ public DestinationDefinitionSpecificationApiController(final SchedulerHandler sc @ExecuteOn(AirbyteTaskExecutors.IO) @Override public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecification(final DestinationDefinitionIdWithWorkspaceId destinationDefinitionIdWithWorkspaceId) { - return ApiHelper.execute(() -> schedulerHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); + return ApiHelper.execute(() -> connectorDefinitionSpecificationHandler.getDestinationSpecification(destinationDefinitionIdWithWorkspaceId)); } @Post("/get_for_destination") @@ -43,7 +43,7 @@ public DestinationDefinitionSpecificationRead getDestinationDefinitionSpecificat @ExecuteOn(AirbyteTaskExecutors.IO) @Override public DestinationDefinitionSpecificationRead getSpecificationForDestinationId(final DestinationIdRequestBody destinationIdRequestBody) { - return ApiHelper.execute(() -> schedulerHandler.getSpecificationForDestinationId(destinationIdRequestBody)); + return ApiHelper.execute(() -> connectorDefinitionSpecificationHandler.getSpecificationForDestinationId(destinationIdRequestBody)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java index 81739ff3660..ac09b7c2d4c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java @@ -6,7 +6,8 @@ import io.airbyte.api.generated.InstanceConfigurationApi; import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.server.handlers.instance_configuration.InstanceConfigurationHandler; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.commons.server.handlers.InstanceConfigurationHandler; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.security.annotation.Secured; @@ -26,7 +27,12 @@ public InstanceConfigurationApiController(final InstanceConfigurationHandler ins @Override @Get public InstanceConfigurationResponse getInstanceConfiguration() { - return instanceConfigurationHandler.getInstanceConfiguration(); + return ApiHelper.execute(instanceConfigurationHandler::getInstanceConfiguration); + } + + @Override + public InstanceConfigurationResponse setupInstanceConfiguration(InstanceConfigurationSetupRequestBody instanceConfigurationSetupRequestBody) { + return ApiHelper.execute(() -> instanceConfigurationHandler.setupInstanceConfiguration(instanceConfigurationSetupRequestBody)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java index 9d687702e0b..9368888733b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiController.java @@ -10,7 +10,7 @@ import io.airbyte.api.model.generated.SourceDefinitionIdWithWorkspaceId; import io.airbyte.api.model.generated.SourceDefinitionSpecificationRead; import io.airbyte.api.model.generated.SourceIdRequestBody; -import io.airbyte.commons.server.handlers.SchedulerHandler; +import io.airbyte.commons.server.handlers.ConnectorDefinitionSpecificationHandler; import io.airbyte.commons.server.scheduling.AirbyteTaskExecutors; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Post; @@ -23,10 +23,10 @@ @Secured(SecurityRule.IS_AUTHENTICATED) public class SourceDefinitionSpecificationApiController implements SourceDefinitionSpecificationApi { - private final SchedulerHandler schedulerHandler; + private final ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler; - public SourceDefinitionSpecificationApiController(final SchedulerHandler schedulerHandler) { - this.schedulerHandler = schedulerHandler; + public SourceDefinitionSpecificationApiController(ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler) { + this.connectorDefinitionSpecificationHandler = connectorDefinitionSpecificationHandler; } @SuppressWarnings("LineLength") @@ -35,7 +35,7 @@ public SourceDefinitionSpecificationApiController(final SchedulerHandler schedul @ExecuteOn(AirbyteTaskExecutors.IO) @Override public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final SourceDefinitionIdWithWorkspaceId sourceDefinitionIdWithWorkspaceId) { - return ApiHelper.execute(() -> schedulerHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId)); + return ApiHelper.execute(() -> connectorDefinitionSpecificationHandler.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId)); } @Post("/get_for_source") @@ -43,7 +43,7 @@ public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(final @ExecuteOn(AirbyteTaskExecutors.IO) @Override public SourceDefinitionSpecificationRead getSpecificationForSourceId(final SourceIdRequestBody sourceIdRequestBody) { - return ApiHelper.execute(() -> schedulerHandler.getSpecificationForSourceId(sourceIdRequestBody)); + return ApiHelper.execute(() -> connectorDefinitionSpecificationHandler.getSpecificationForSourceId(sourceIdRequestBody)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java index 11d241ee5e2..d55acdbca69 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/StateApiController.java @@ -42,6 +42,7 @@ public ConnectionState createOrUpdateState(final ConnectionStateCreateOrUpdate c @Post("/create_or_update_safe") @Secured({EDITOR}) + @SecuredWorkspace @ExecuteOn(AirbyteTaskExecutors.IO) @Override public ConnectionState createOrUpdateStateSafe(final ConnectionStateCreateOrUpdate connectionStateCreateOrUpdate) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java index 94d0ef194eb..58705529ba0 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java @@ -15,6 +15,7 @@ import io.airbyte.api.model.generated.UserIdRequestBody; import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserReadList; import io.airbyte.commons.auth.SecuredUser; @@ -28,10 +29,8 @@ import io.micronaut.security.rules.SecurityRule; /** - * This class is migrated from cloud-server UserApiController - * {@link io.airbyte.cloud.server.apis.UserApiController}. - * - * TODO: migrate all User endpoints (including some endpoints in WebBackend API) from Cloud to OSS. + * User related APIs. TODO: migrate all User endpoints (including some endpoints in WebBackend API) + * from Cloud to OSS. */ @SuppressWarnings("MissingJavadocType") @Controller("/api/v1/users") @@ -104,4 +103,13 @@ public WorkspaceUserReadList listUsersInWorkspace(@Body final WorkspaceIdRequest return ApiHelper.execute(() -> userHandler.listUsersInWorkspace(workspaceIdRequestBody)); } + // TODO: Update permission to instance admin once the permission PR is merged. + @Post("/list_instance_admins") + @Secured({READER}) + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public UserWithPermissionInfoReadList listInstanceAdminUsers() { + return ApiHelper.execute(() -> userHandler.listInstanceAdminUsers()); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index 9800165f874..e1a11df154a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -11,6 +11,7 @@ import io.airbyte.api.generated.WorkspaceApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; +import io.airbyte.api.model.generated.ListWorkspacesByUserRequestBody; import io.airbyte.api.model.generated.ListWorkspacesInOrganizationRequestBody; import io.airbyte.api.model.generated.SlugRequestBody; import io.airbyte.api.model.generated.WorkspaceCreate; @@ -149,4 +150,12 @@ public WorkspaceReadList listWorkspacesInOrganization(@Body final ListWorkspaces return ApiHelper.execute(() -> workspacesHandler.listWorkspacesInOrganization(request)); } + @Post("/list_by_user_id") + @Secured({READER}) + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public WorkspaceReadList listWorkspacesByUser(@Body final ListWorkspacesByUserRequestBody request) { + return ApiHelper.execute(() -> workspacesHandler.listWorkspacesByUser(request)); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index 9bd931f4e9b..63238e06e4c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -19,6 +19,9 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.DefaultJobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -127,6 +130,12 @@ public FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); } + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.SERVER); + return MetricClientFactory.getMetricClient(); + } + @Singleton @Named("workspaceRoot") public Path workspaceRoot(@Value("${airbyte.workspace.root}") final String workspaceRoot) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java index 5d4b661779b..f9a9a986a6f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java @@ -19,7 +19,9 @@ import io.airbyte.db.instance.DatabaseConstants; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; import io.micronaut.flyway.FlywayConfigurationProperties; @@ -94,6 +96,11 @@ public JobPersistence jobPersistence(@Named("configDatabase") final Database job return new DefaultJobPersistence(jobDatabase); } + @Singleton + public MetadataPersistence metadataPersistence(@Named("configDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + @Singleton public PermissionPersistence permissionPersistence(@Named("configDatabase") final Database configDatabase) { return new PermissionPersistence(configDatabase); diff --git a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java index 384cfb28a02..586bedddf3a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java +++ b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java @@ -8,9 +8,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.commons.auth.AuthRole; +import io.airbyte.commons.auth.OrganizationAuthRole; import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.license.annotation.RequiresAirbyteProEnabled; +import io.airbyte.commons.server.support.AuthenticationHeaderResolver; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.persistence.PermissionPersistence; import io.micrometer.common.util.StringUtils; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpRequest; @@ -22,9 +27,17 @@ import io.micronaut.security.authentication.AuthenticationException; import io.micronaut.security.token.validator.TokenValidator; import jakarta.inject.Singleton; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.reactivestreams.Publisher; import reactor.core.publisher.Mono; @@ -36,16 +49,28 @@ @Slf4j @Singleton @RequiresAirbyteProEnabled -@SuppressWarnings({"PMD.PreserveStackTrace", "PMD.UseTryWithResources"}) +@SuppressWarnings({"PMD.PreserveStackTrace", "PMD.UseTryWithResources", "PMD.UnusedFormalParameter", "PMD.UnusedPrivateMethod"}) public class KeycloakTokenValidator implements TokenValidator { private final HttpClient client; private final AirbyteKeycloakConfiguration keycloakConfiguration; + private final AuthenticationHeaderResolver headerResolver; + private final PermissionPersistence permissionPersistence; + + private static final Map WORKSPACE_PERMISSION_TYPE_TO_AUTH_ROLE = Map.of(PermissionType.WORKSPACE_ADMIN, AuthRole.ADMIN, + PermissionType.WORKSPACE_EDITOR, AuthRole.EDITOR, PermissionType.WORKSPACE_READER, AuthRole.READER); + private static final Map ORGANIZATION_PERMISSION_TYPE_TO_AUTH_ROLE = + Map.of(PermissionType.ORGANIZATION_ADMIN, OrganizationAuthRole.ORGANIZATION_ADMIN, PermissionType.ORGANIZATION_EDITOR, + OrganizationAuthRole.ORGANIZATION_EDITOR, PermissionType.ORGANIZATION_READER, OrganizationAuthRole.ORGANIZATION_READER); public KeycloakTokenValidator(final HttpClient httpClient, - final AirbyteKeycloakConfiguration keycloakConfiguration) { + final AirbyteKeycloakConfiguration keycloakConfiguration, + final AuthenticationHeaderResolver headerResolver, + final PermissionPersistence permissionPersistence) { this.client = httpClient; this.keycloakConfiguration = keycloakConfiguration; + this.headerResolver = headerResolver; + this.permissionPersistence = permissionPersistence; } @Override @@ -53,7 +78,7 @@ public Publisher validateToken(final String token, final HttpReq return validateTokenWithKeycloak(token) .flatMap(valid -> { if (valid) { - return Mono.just(getAuthentication(token)); + return Mono.just(getAuthentication(token, request)); } else { // pass to the next validator, if one exists log.warn("Token was not a valid Keycloak token: {}", token); @@ -62,7 +87,7 @@ public Publisher validateToken(final String token, final HttpReq }); } - private Authentication getAuthentication(final String token) { + private Authentication getAuthentication(final String token, final HttpRequest request) { final String[] tokenParts = token.split("\\."); final String payload = tokenParts[1]; @@ -76,7 +101,10 @@ private Authentication getAuthentication(final String token) { if (StringUtils.isNotBlank(userId)) { log.debug("Fetching roles for user '{}'...", userId); - final Collection roles = getRoles(userId); + // For now, give all valid Keycloak users instance_admin, until we actually create an Airbyte User + // with permissions for such logins. + final Collection roles = getInstanceAdminRoles(); + // final Collection roles = getRoles(userId, request); log.debug("Authenticating user '{}' with roles {}...", userId, roles); return Authentication.build(userId, roles); } else { @@ -88,13 +116,56 @@ private Authentication getAuthentication(final String token) { } } - /** - * For now, we are granting ADMIN to all authenticated users. This will change with the introduction - * of RBAC. - */ - private Collection getRoles(final String userId) { - log.debug("Granting ADMIN role to user {}", userId); - return AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + private Collection getInstanceAdminRoles() { + Set roles = new HashSet<>(); + roles.addAll(AuthRole.buildAuthRolesSet(AuthRole.ADMIN)); + roles.addAll(OrganizationAuthRole.buildOrganizationAuthRolesSet(OrganizationAuthRole.ORGANIZATION_ADMIN)); + return roles; + } + + private Collection getRoles(final String userId, final HttpRequest request) { + Map headerMap = request.getHeaders().asMap(String.class, String.class); + + // We will check for permissions over organization and workspace + final List workspaceIds = headerResolver.resolveWorkspace(headerMap); + final List organizationIds = headerResolver.resolveOrganization(headerMap); + + Set roles = new HashSet<>(); + roles.add(AuthRole.AUTHENTICATED_USER.toString()); + + // Find the minimum permission for workspace + if (workspaceIds != null && !workspaceIds.isEmpty()) { + Optional minAuthRoleOptional = workspaceIds.stream() + .map(workspaceId -> { + try { + return permissionPersistence.findPermissionTypeForUserAndWorkspace(workspaceId, userId, AuthProvider.KEYCLOAK); + } catch (IOException ex) { + log.error("Failed to get permission for user {} and workspaces {}", userId, workspaceId, ex); + throw new RuntimeException(ex); + } + }) + .map(permissionType -> WORKSPACE_PERMISSION_TYPE_TO_AUTH_ROLE.get(permissionType)) + .min(Comparator.comparingInt(AuthRole::getAuthority)); + AuthRole authRole = minAuthRoleOptional.orElse(AuthRole.NONE); + roles.addAll(AuthRole.buildAuthRolesSet(authRole)); + } + if (organizationIds != null && !organizationIds.isEmpty()) { + Optional minAuthRoleOptional = organizationIds.stream() + .map(organizationId -> { + try { + return permissionPersistence.findPermissionTypeForUserAndOrganization(organizationId, userId, AuthProvider.KEYCLOAK); + } catch (IOException ex) { + log.error("Failed to get permission for user {} and organization {}", userId, organizationId, ex); + throw new RuntimeException(ex); + } + }) + .map(permissionType -> ORGANIZATION_PERMISSION_TYPE_TO_AUTH_ROLE.get(permissionType)) + .min(Comparator.comparingInt(OrganizationAuthRole::getAuthority)); + OrganizationAuthRole authRole = minAuthRoleOptional.orElse(OrganizationAuthRole.NONE); + roles.addAll(OrganizationAuthRole.buildOrganizationAuthRolesSet(authRole)); + } + + return roles; } private Mono validateTokenWithKeycloak(final String token) { diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 85e6ea1ce36..b27e1464d9f 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -23,12 +23,14 @@ micronaut: allowed-origins-regex: - ^.*$ netty: + aggregator: + max-content-length: 52428800 # 50MB access-logger: enabled: ${HTTP_ACCESS_LOG_ENABLED:true} idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} airbyte: - edition: ${AIRBYTE_EDITION:community} + edition: ${AIRBYTE_EDITION:COMMUNITY} shutdown: delay_ms: 20000 cloud: diff --git a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java index 723931057e2..54d544553e4 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java @@ -11,6 +11,10 @@ import io.airbyte.commons.auth.AuthRole; import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.server.support.AuthenticationHeaderResolver; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.persistence.PermissionPersistence; import io.airbyte.server.pro.KeycloakTokenValidator; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpRequest; @@ -21,6 +25,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,9 +42,14 @@ class KeycloakTokenValidatorTest { private static final String URI_PATH = "/some/path"; private static final Collection DEFAULT_ROLES = AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID ORGANIZATION_ID = UUID.randomUUID(); + private KeycloakTokenValidator keycloakTokenValidator; private HttpClient httpClient; private AirbyteKeycloakConfiguration keycloakConfiguration; + private AuthenticationHeaderResolver headerResolver; + private PermissionPersistence permissionPersistence; @BeforeEach void setUp() { @@ -45,8 +57,10 @@ void setUp() { keycloakConfiguration = mock(AirbyteKeycloakConfiguration.class); when(keycloakConfiguration.getKeycloakUserInfoEndpoint()).thenReturn(LOCALHOST + URI_PATH); + headerResolver = mock(AuthenticationHeaderResolver.class); + permissionPersistence = mock(PermissionPersistence.class); - keycloakTokenValidator = new KeycloakTokenValidator(httpClient, keycloakConfiguration); + keycloakTokenValidator = new KeycloakTokenValidator(httpClient, keycloakConfiguration, headerResolver, permissionPersistence); } @AfterEach @@ -55,7 +69,7 @@ void tearDown() { } @Test - void testValidateToken() throws URISyntaxException { + void testValidateToken() throws Exception { final URI uri = new URI(LOCALHOST + URI_PATH); final String accessToken = """ eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIwM095c3pkWmNrZFd6Mk84d0ZFRkZVblJPLVJrN1lGLWZzRm1kWG1Q @@ -95,8 +109,19 @@ void testValidateToken() throws URISyntaxException { final Publisher responsePublisher = keycloakTokenValidator.validateToken(accessTokenWithoutNewline, httpRequest); + when(headerResolver.resolveWorkspace(any())).thenReturn(List.of(WORKSPACE_ID)); + when(headerResolver.resolveOrganization(any())).thenReturn(List.of(ORGANIZATION_ID)); + when(permissionPersistence.findPermissionTypeForUserAndWorkspace(WORKSPACE_ID, expectedUserId, AuthProvider.KEYCLOAK)) + .thenReturn(PermissionType.WORKSPACE_ADMIN); + when(permissionPersistence.findPermissionTypeForUserAndOrganization(ORGANIZATION_ID, expectedUserId, AuthProvider.KEYCLOAK)) + .thenReturn(PermissionType.ORGANIZATION_READER); + + // TODO: enable this once we are ready to enable getRoles function in KeycloakTokenValidator. + // Set expectedResult = Set.of("ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); + Set expectedResult = Set.of("ORGANIZATION_ADMIN", "ORGANIZATION_EDITOR", "ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); + StepVerifier.create(responsePublisher) - .expectNextMatches(r -> matchSuccessfulResponse(r, expectedUserId, DEFAULT_ROLES)) + .expectNextMatches(r -> matchSuccessfulResponse(r, expectedUserId, expectedResult)) .verifyComplete(); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/BaseControllerTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/BaseControllerTest.java index 4e72801de7e..0ad3b4f69a9 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/BaseControllerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/BaseControllerTest.java @@ -10,6 +10,7 @@ import io.airbyte.commons.server.handlers.ActorDefinitionVersionHandler; import io.airbyte.commons.server.handlers.AttemptHandler; import io.airbyte.commons.server.handlers.ConnectionsHandler; +import io.airbyte.commons.server.handlers.ConnectorDefinitionSpecificationHandler; import io.airbyte.commons.server.handlers.DestinationDefinitionsHandler; import io.airbyte.commons.server.handlers.DestinationHandler; import io.airbyte.commons.server.handlers.HealthCheckHandler; @@ -192,6 +193,15 @@ SchedulerHandler mmSchedulerHandler() { return schedulerHandler; } + ConnectorDefinitionSpecificationHandler connectorDefinitionSpecificationHandler = Mockito.mock(ConnectorDefinitionSpecificationHandler.class); + + @MockBean(ConnectorDefinitionSpecificationHandler.class) + @Replaces(ConnectorDefinitionSpecificationHandler.class) + + ConnectorDefinitionSpecificationHandler mmConnectorDefinitionSpecificationHandler() { + return connectorDefinitionSpecificationHandler; + } + SourceDefinitionsHandler sourceDefinitionsHandler = Mockito.mock(SourceDefinitionsHandler.class); @MockBean(SourceDefinitionsHandler.class) diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiTest.java index 3c6b889ec96..9f8ff300880 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/DestinationDefinitionSpecificationApiTest.java @@ -25,7 +25,7 @@ class DestinationDefinitionSpecificationApiTest extends BaseControllerTest { @Test void testCheckConnectionToDestination() throws JsonValidationException, ConfigNotFoundException, IOException { - Mockito.when(schedulerHandler.getDestinationSpecification(Mockito.any())) + Mockito.when(connectorDefinitionSpecificationHandler.getDestinationSpecification(Mockito.any())) .thenReturn(new DestinationDefinitionSpecificationRead()) .thenThrow(new ConfigNotFoundException("", "")); final String path = "/api/v1/destination_definition_specifications/get"; diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java index a32da5b2e3b..ec2e7066bac 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java @@ -7,7 +7,9 @@ import static org.mockito.Mockito.when; import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.server.handlers.instance_configuration.InstanceConfigurationHandler; +import io.airbyte.commons.server.handlers.InstanceConfigurationHandler; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Replaces; import io.micronaut.context.annotation.Requires; import io.micronaut.context.env.Environment; @@ -16,6 +18,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.test.annotation.MockBean; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import java.io.IOException; import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.Mockito; @@ -40,11 +43,19 @@ InstanceConfigurationHandler mmInstanceConfigurationHandler() { static String PATH = "/api/v1/instance_configuration"; @Test - void testGetInstanceConfiguration() { + void testGetInstanceConfiguration() throws ConfigNotFoundException, IOException { when(instanceConfigurationHandler.getInstanceConfiguration()) .thenReturn(new InstanceConfigurationResponse()); testEndpointStatus(HttpRequest.GET(PATH), HttpStatus.OK); } + @Test + void testSetupInstanceConfiguration() throws ConfigNotFoundException, IOException, JsonValidationException { + when(instanceConfigurationHandler.setupInstanceConfiguration(Mockito.any())) + .thenReturn(new InstanceConfigurationResponse()); + + testEndpointStatus(HttpRequest.POST(PATH + "/setup", new InstanceConfigurationResponse()), HttpStatus.OK); + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiTest.java index 55b0aa9d777..0cd6b9a8c1c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/SourceDefinitionSpecificationApiTest.java @@ -26,7 +26,7 @@ class SourceDefinitionSpecificationApiTest extends BaseControllerTest { @Test void testCreateCustomSourceDefinition() throws IOException, JsonValidationException, ConfigNotFoundException { - Mockito.when(schedulerHandler.getSourceDefinitionSpecification(Mockito.any())) + Mockito.when(connectorDefinitionSpecificationHandler.getSourceDefinitionSpecification(Mockito.any())) .thenReturn(new SourceDefinitionSpecificationRead()) .thenThrow(new ConfigNotFoundException("", "")); final String path = "/api/v1/source_definition_specifications/get"; diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java index cd228d40a1b..52f1767170f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java @@ -11,6 +11,7 @@ import io.airbyte.api.model.generated.UserIdRequestBody; import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserReadList; import io.airbyte.commons.json.Jsons; @@ -99,4 +100,14 @@ void testListUsersInWorkspace() throws Exception { HttpStatus.OK); } + @Test + void testListInstanceAdminUsers() throws Exception { + Mockito.when(userHandler.listInstanceAdminUsers()) + .thenReturn(new UserWithPermissionInfoReadList()); + final String path = "/api/v1/users/list_instance_admin"; + testEndpointStatus( + HttpRequest.POST(path, Jsons.emptyObject()), + HttpStatus.OK); + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java index b8b39ceaabf..be36110d313 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java @@ -26,7 +26,7 @@ class WorkspaceApiTest extends BaseControllerTest { @Test - void testCreateWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testCreateWorkspace() throws JsonValidationException, IOException { Mockito.when(workspacesHandler.createWorkspace(Mockito.any())) .thenReturn(new WorkspaceRead()); final String path = "/api/v1/workspaces/create"; @@ -64,7 +64,7 @@ void testGetWorkspace() throws JsonValidationException, ConfigNotFoundException, } @Test - void testGetBySlugWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetBySlugWorkspace() throws ConfigNotFoundException, IOException { Mockito.when(workspacesHandler.getWorkspaceBySlug(Mockito.any())) .thenReturn(new WorkspaceRead()) .thenThrow(new ConfigNotFoundException("", "")); @@ -78,7 +78,7 @@ void testGetBySlugWorkspace() throws JsonValidationException, ConfigNotFoundExce } @Test - void testListWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testListWorkspace() throws JsonValidationException, IOException { Mockito.when(workspacesHandler.listWorkspaces()) .thenReturn(new WorkspaceReadList()); final String path = "/api/v1/workspaces/list"; @@ -130,7 +130,7 @@ void testUpdateWorkspaceName() throws JsonValidationException, ConfigNotFoundExc } @Test - void testGetWorkspaceByConnectionId() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetWorkspaceByConnectionId() throws ConfigNotFoundException { Mockito.when(workspacesHandler.getWorkspaceByConnectionId(Mockito.any())) .thenReturn(new WorkspaceRead()) .thenThrow(new ConfigNotFoundException("", "")); diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java index 306c58ab404..85465d3a714 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java @@ -785,14 +785,16 @@ public SourceDefinitionRead createE2eSourceDefinition(final UUID workspaceId) { return sourceDefinitionRead; } - public DestinationDefinitionRead createE2eDestinationDefinition(final UUID workspaceId) throws ApiException { - return apiClient.getDestinationDefinitionApi().createCustomDestinationDefinition(new CustomDestinationDefinitionCreate() - .workspaceId(workspaceId) - .destinationDefinition(new DestinationDefinitionCreate() - .name("E2E Test Destination") - .dockerRepository("airbyte/destination-e2e-test") - .dockerImageTag(DESTINATION_E2E_TEST_CONNECTOR_VERSION) - .documentationUrl(URI.create("https://example.com")))); + public DestinationDefinitionRead createE2eDestinationDefinition(final UUID workspaceId) throws Exception { + return AirbyteApiClient.retryWithJitterThrows(() -> apiClient.getDestinationDefinitionApi() + .createCustomDestinationDefinition(new CustomDestinationDefinitionCreate() + .workspaceId(workspaceId) + .destinationDefinition(new DestinationDefinitionCreate() + .name("E2E Test Destination") + .dockerRepository("airbyte/destination-e2e-test") + .dockerImageTag(DESTINATION_E2E_TEST_CONNECTOR_VERSION) + .documentationUrl(URI.create("https://example.com")))), + "create destination definition", 10, 60, 3); } public SourceRead createPostgresSource() { diff --git a/airbyte-tests/build.gradle.kts b/airbyte-tests/build.gradle.kts index 0e54d02d56a..662ee82d51a 100644 --- a/airbyte-tests/build.gradle.kts +++ b/airbyte-tests/build.gradle.kts @@ -64,6 +64,8 @@ fun registerTestSuite(name: String, type: String, dirName: String, deps: JvmComp events = setOf(TestLogEvent.PASSED, TestLogEvent.FAILED) } shouldRunAfter(suites.named("test")) + // Ensure they re-run since these are integration tests. + outputs.upToDateWhen { false } } } } diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index 05e1a99b3c7..79dc8c4edfd 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -46,13 +46,10 @@ import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; import java.net.URISyntaxException; -import java.sql.SQLException; import java.util.List; import java.util.UUID; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -121,21 +118,14 @@ static void end() { testHarness.stopDbAndContainers(); } - @BeforeEach - void setup() throws URISyntaxException, IOException, SQLException { - testHarness.setup(); - } - - @AfterEach - void tearDown() { - testHarness.cleanup(); - } - @SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") - @Test + @RetryingTest(3) void testManualSync() throws Exception { + testHarness.setup(); + final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); + final SourceDiscoverSchemaRead discoverResult = testHarness.discoverSourceSchemaWithId(sourceId); final AirbyteCatalog catalog = discoverResult.getCatalog(); final SyncMode syncMode = SyncMode.FULL_REFRESH; @@ -153,8 +143,9 @@ void testManualSync() throws Exception { waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); Asserts.assertSourceAndDestinationDbRawRecordsInSync(testHarness.getSourceDatabase(), testHarness.getDestinationDatabase(), PUBLIC_SCHEMA_NAME, conn.getNamespaceFormat(), false, false); - Asserts.assertStreamStatuses(apiClient, workspaceId, conn.getConnectionId(), connectionSyncRead, StreamStatusRunState.COMPLETE, - StreamStatusJobType.SYNC); + Asserts.assertStreamStatuses(apiClient, workspaceId, connectionId, connectionSyncRead, StreamStatusRunState.COMPLETE, StreamStatusJobType.SYNC); + + testHarness.cleanup(); } @RetryingTest(3) diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index 35a661b0bea..588b0632ddd 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -387,7 +387,7 @@ void testDiscoverSourceSchema() throws ApiException { final AirbyteStreamConfiguration expectedStreamConfig = new AirbyteStreamConfiguration() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) - .destinationSyncMode(DestinationSyncMode.APPEND) + .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) .aliasName(STREAM_NAME.replace(".", "_")) .selected(true) diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java index 3b610c56fdd..45d34baf122 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java @@ -16,16 +16,13 @@ import io.airbyte.api.client2.model.generated.SourceDefinitionCreate; import io.airbyte.api.client2.model.generated.SourceDefinitionIdRequestBody; import io.airbyte.api.client2.model.generated.SourceDefinitionRead; -import io.airbyte.test.utils.AcceptanceTestHarness; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.sql.SQLException; +import java.time.Duration; import java.util.UUID; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; +import okhttp3.OkHttpClient; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.params.ParameterizedTest; @@ -37,30 +34,17 @@ class VersioningAcceptanceTests { private static AirbyteApiClient2 apiClient2; private static UUID workspaceId; - private static AcceptanceTestHarness testHarness; - @BeforeAll - static void init() throws URISyntaxException, IOException, InterruptedException { - testHarness = new AcceptanceTestHarness(null, workspaceId); - RetryPolicy policy = RetryPolicy.ofDefaults(); - apiClient2 = new AirbyteApiClient2("http://localhost:8001/api", policy); - - workspaceId = apiClient2.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); - } + static void init() throws IOException { + RetryPolicy policy = RetryPolicy.builder() + .handle(Throwable.class) + .withMaxAttempts(5) + .withBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)).build(); - @AfterAll - static void afterAll() { - testHarness.stopDbAndContainers(); - } - - @BeforeEach - void setup() throws SQLException, URISyntaxException, IOException { - testHarness.setup(); - } + OkHttpClient client = new OkHttpClient.Builder().readTimeout(Duration.ofSeconds(20)).build(); + apiClient2 = new AirbyteApiClient2("http://localhost:8001/api", policy, client); - @AfterEach - void tearDown() { - testHarness.cleanup(); + workspaceId = apiClient2.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); } @ParameterizedTest diff --git a/airbyte-tests/src/test-acceptance/resources/junit-platform.properties b/airbyte-tests/src/test-acceptance/resources/junit-platform.properties index 19747320c8f..9e63e3e8955 100644 --- a/airbyte-tests/src/test-acceptance/resources/junit-platform.properties +++ b/airbyte-tests/src/test-acceptance/resources/junit-platform.properties @@ -1,2 +1,2 @@ -#junit.jupiter.execution.parallel.enabled=true -#junit.jupiter.execution.parallel.mode.classes.default=concurrent +junit.jupiter.execution.parallel.enabled=true +junit.jupiter.execution.parallel.mode.classes.default=concurrent diff --git a/airbyte-webapp/.eslintLegacyFolderStructure.js b/airbyte-webapp/.eslintLegacyFolderStructure.js index f1cd97bad69..bfae59d6512 100644 --- a/airbyte-webapp/.eslintLegacyFolderStructure.js +++ b/airbyte-webapp/.eslintLegacyFolderStructure.js @@ -43,7 +43,6 @@ module.exports = [ "src/hooks/services/Modal/types.ts", "src/hooks/services/Modal/ModalService.test.tsx", "src/hooks/services/Modal/index.ts", - "src/hooks/services/useConnectionHook.tsx", "src/hooks/services/useRequestErrorHandler.tsx", "src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx", "src/hooks/services/ConnectionForm/ConnectionFormService.tsx", diff --git a/airbyte-webapp/.eslintrc.js b/airbyte-webapp/.eslintrc.js index e68a44e24e3..8bed9692520 100644 --- a/airbyte-webapp/.eslintrc.js +++ b/airbyte-webapp/.eslintrc.js @@ -106,6 +106,12 @@ module.exports = { "error", { paths: [ + { + name: "react-use", + importNames: ["useLocalStorage"], + message: + 'Please use our wrapped version of this hook with `import { useLocalStorage } from "core/utils/useLocalStorage";` instead.', + }, { name: "lodash", message: 'Please use `import [function] from "lodash/[function]";` instead.', diff --git a/airbyte-webapp/.storybook/main.ts b/airbyte-webapp/.storybook/main.ts index ba47e941ac3..769c459b231 100644 --- a/airbyte-webapp/.storybook/main.ts +++ b/airbyte-webapp/.storybook/main.ts @@ -5,6 +5,7 @@ const config: StorybookConfig = { framework: "@storybook/react-vite", stories: ["../src/**/*.stories.@(ts|tsx)", "../src/**/*.docs.mdx"], addons: [ + "storybook-dark-mode", "@storybook/addon-links", "@storybook/addon-essentials", { diff --git a/airbyte-webapp/.storybook/preview.ts b/airbyte-webapp/.storybook/preview.ts index 1e86bf0d54b..d18c59948f0 100644 --- a/airbyte-webapp/.storybook/preview.ts +++ b/airbyte-webapp/.storybook/preview.ts @@ -4,5 +4,14 @@ import "../public/index.css"; import "../src/scss/global.scss"; import "../src/dayjs-setup"; -export const parameters = {}; +export const parameters = { + darkMode: { + stylePreview: true, + darkClass: ["airbyteThemeDark"], + lightClass: ["airbyteThemeLight"], + }, + backgrounds: { + disable: true, + }, +}; export const decorators = [withProviders]; diff --git a/airbyte-webapp/.storybook/withProvider.tsx b/airbyte-webapp/.storybook/withProvider.tsx index 56dc8c6f90c..d0535512278 100644 --- a/airbyte-webapp/.storybook/withProvider.tsx +++ b/airbyte-webapp/.storybook/withProvider.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { Decorator } from "@storybook/react"; import { MemoryRouter } from "react-router-dom"; import { IntlProvider } from "react-intl"; @@ -32,7 +33,7 @@ const queryClient = new QueryClient({ }, }); -export const withProviders = (getStory) => ( +export const withProviders = (getStory: Parameters[0]) => ( diff --git a/airbyte-webapp/README.md b/airbyte-webapp/README.md index eebc734c993..b29270d6978 100644 --- a/airbyte-webapp/README.md +++ b/airbyte-webapp/README.md @@ -52,12 +52,15 @@ src/ ├ area/ # Code for a specific domain of the webapp │ ├ connection/ │ │ ├ services/ # Services for this area +│ │ ├ types/ # Types and enums that must be available widely and are not explicitly +│ │ │ # belonging to a component or util (e.g. prop types) in which case they should +│ │ │ # be living within that component/util file. │ │ ├ utils/ # Utils for this area │ │ └ components/ # Components related to this area or for pages specific to this area │ ├ connector/ # Has the same services/, utils/, components/ structure │ ├ connectorBuilder/ │ ├ settings/ -│ └ workspaces/ +│ └ workspace/ └ cloud/ # Cloud specific code (following a similar structure as above) ``` diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index a335aeed378..9c4423a5c85 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -28,23 +28,7 @@ node { distBaseUrl = "https://nodejs.org/dist" } -tasks.register("validateLockFiles") { - description "Validate only a pnpm-lock.yaml lock file exists" - inputs.files "pnpm-lock.yaml", "package-lock.json", "yarn.lock" - - // The validateLockFiles has no outputs, thus we always treat the outputs up to date - // as long as the inputs have not changed - outputs.upToDateWhen { true } - - doLast { - assert file("pnpm-lock.yaml").exists() - assert !file("package-lock.json").exists() - assert !file("yarn.lock").exists() - } -} - tasks.named("pnpmInstall") { - dependsOn tasks.named("validateLockFiles") // Add patches folder to inputs of pnpmInstall task, since it has pnpm-lock.yml as an output // thus wouldn't rerun in case a patch get changed inputs.dir "patches" @@ -125,35 +109,22 @@ tasks.register("cloudE2eTest", PnpmTask) { outputs.upToDateWhen { false } } -tasks.register("licenseCheck", PnpmTask) { - dependsOn tasks.named("pnpmInstall") - - args = ['run', 'license-check'] - - inputs.files nodeModules - inputs.file 'package.json' - inputs.file 'scripts/license-check.js' - - // The licenseCheck has no outputs, thus we always treat the outputs up to date - // as long as the inputs have not changed - outputs.upToDateWhen { true } -} - -tasks.register("validateLinks", PnpmTask) { - dependsOn tasks.named("pnpmInstall") - - args = ['run', 'validate-links'] - - inputs.file 'scripts/validate-links.ts' - inputs.file 'src/core/utils/links.ts' - - // Since the output of this task depends on availability of URLs - // we never want to treat it as "up-to-date" on CI and always want to run it - // but running locally we treat it as up-to-date just depending on its inputs - outputs.upToDateWhen { - System.getenv("CI") === null - } -} +// +//tasks.register("validateLinks", PnpmTask) { +// dependsOn tasks.named("pnpmInstall") +// +// args = ['run', 'validate-links'] +// +// inputs.file 'scripts/validate-links.ts' +// inputs.file 'src/core/utils/links.ts' +// +// // Since the output of this task depends on availability of URLs +// // we never want to treat it as "up-to-date" on CI and always want to run it +// // but running locally we treat it as up-to-date just depending on its inputs +// outputs.upToDateWhen { +// System.getenv("CI") === null +// } +//} tasks.register("buildStorybook", PnpmTask) { dependsOn tasks.named("pnpmInstall") @@ -189,10 +160,9 @@ tasks.register("copyNginx", Copy) { // Those tasks should be run as part of the "check" task tasks.named("check") { - dependsOn tasks.named("validateLinks"), tasks.named("licenseCheck"), tasks.named("test") + dependsOn /*tasks.named("validateLinks"),*/ tasks.named("test") } - tasks.named("build") { dependsOn tasks.named("buildStorybook") } diff --git a/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts b/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts index ff2766766f6..afd0cbcd03e 100644 --- a/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts +++ b/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts @@ -1,6 +1,6 @@ import { FeatureItem } from "@src/core/services/features"; -describe("AllowDBTCloudIntegration", () => { +describe.skip("AllowDBTCloudIntegration", () => { beforeEach(() => { cy.login(); cy.selectWorkspace(); diff --git a/airbyte-webapp/cypress/commands/common.ts b/airbyte-webapp/cypress/commands/common.ts index 004ca97dec7..e5a4b758e60 100644 --- a/airbyte-webapp/cypress/commands/common.ts +++ b/airbyte-webapp/cypress/commands/common.ts @@ -26,10 +26,6 @@ export const clearApp = () => { cy.clearCookies(); }; -export const fillEmail = (email: string) => { - cy.get("input[name=email]").type(email); -}; - // useful for ensuring that a name is unique from one test run to the next export const appendRandomString = (string: string) => { const randomString = Math.random().toString(36).substring(2, 10); diff --git a/airbyte-webapp/cypress/commands/connector.ts b/airbyte-webapp/cypress/commands/connector.ts index 5e84846e9e7..1d59cd3b65e 100644 --- a/airbyte-webapp/cypress/commands/connector.ts +++ b/airbyte-webapp/cypress/commands/connector.ts @@ -11,6 +11,7 @@ import { enterPokemonName, enterSchema, openOptionalFields, + selectXmin, } from "pages/createConnectorPage"; export const fillPostgresForm = ( @@ -37,6 +38,7 @@ export const fillPostgresForm = ( enterSchema(schema); enterUsername(username); enterPassword(password); + selectXmin(); }; export const fillPokeAPIForm = (name: string, pokeName: string) => { diff --git a/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts b/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts index ac2bbcbb1fb..042ef13070a 100644 --- a/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts +++ b/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts @@ -57,28 +57,27 @@ const modifyAccountsTableInterceptHandler: RouteHandler = (request) => { }; const saveConnectionAndAssertStreams = ( - ...expectedSyncModes: Array<{ namespace: string; name: string; config: Partial }> + expectedSyncMode: { namespace: string; name: string; config: Partial }, + { expectModal = true }: { expectModal?: boolean } | undefined = {} ) => { replicationPage - .saveChangesAndHandleResetModal({ interceptUpdateHandler: modifyAccountsTableInterceptHandler }) + .saveChangesAndHandleResetModal({ interceptUpdateHandler: modifyAccountsTableInterceptHandler, expectModal }) .then((connection) => { - expectedSyncModes.forEach((expected) => { - const stream = connection.syncCatalog.streams.find( - ({ stream }) => stream?.namespace === expected.namespace && stream.name === expected.name - ); - - expect(stream).to.exist; - expect(stream?.config).to.contain({ - syncMode: expected.config.syncMode, - destinationSyncMode: expected.config.destinationSyncMode, - }); - if (expected.config.cursorField) { - expect(stream?.config?.cursorField).to.eql(expected.config.cursorField); - } - if (expected.config.primaryKey) { - expect(stream?.config?.cursorField).to.eql(expected.config.cursorField); - } + const stream = connection.syncCatalog.streams.find( + ({ stream }) => stream?.namespace === expectedSyncMode.namespace && stream.name === expectedSyncMode.name + ); + + expect(stream).to.exist; + expect(stream?.config).to.contain({ + syncMode: expectedSyncMode.config.syncMode, + destinationSyncMode: expectedSyncMode.config.destinationSyncMode, }); + if (expectedSyncMode.config.cursorField) { + expect(stream?.config?.cursorField).to.eql(expectedSyncMode.config.cursorField); + } + if (expectedSyncMode.config.primaryKey) { + expect(stream?.config?.cursorField).to.eql(expectedSyncMode.config.cursorField); + } }); }; @@ -155,14 +154,17 @@ describe("Connection - sync modes", () => { streamDetails.close(); // Save - saveConnectionAndAssertStreams({ - namespace: "public", - name: "users", - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, + saveConnectionAndAssertStreams( + { + namespace: "public", + name: "users", + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, }, - }); + { expectModal: false } + ); // Confirm after save usersStreamRow.hasSelectedSyncMode(SyncMode.full_refresh, DestinationSyncMode.overwrite); diff --git a/airbyte-webapp/cypress/e2e/onboarding.cy.ts b/airbyte-webapp/cypress/e2e/onboarding.cy.ts index 0bbdb861abb..af1e12c50be 100644 --- a/airbyte-webapp/cypress/e2e/onboarding.cy.ts +++ b/airbyte-webapp/cypress/e2e/onboarding.cy.ts @@ -1,10 +1,15 @@ -import { submitButtonClick, fillEmail } from "commands/common"; +import { submitButtonClick } from "commands/common"; const OSS_SECURITY_CHECK_URL = "https://oss.airbyte.com/security-check"; +export const fillSetupForm = () => { + cy.get("input[name=email]").type("test-email-onboarding@test-onboarding-domain.com"); + cy.get("input[name=organizationName]").type("ACME Corp"); +}; + describe("Setup actions", () => { beforeEach(() => { - cy.intercept("POST", "/api/v1/workspaces/get", (req) => { + cy.intercept("GET", "/api/v1/instance_configuration", (req) => { req.continue((res) => { res.body.initialSetupComplete = false; res.send(res.body); @@ -21,7 +26,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("[data-testid=securityCheckRunning]").should("be.visible"); cy.get("button[type=submit]").should("be.disabled"); @@ -35,7 +40,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("button[type=submit]").should("be.enabled"); }); @@ -50,7 +55,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("button[type=submit]").should("be.disabled"); cy.get("[data-testid=advancedOptions]").click(); @@ -69,7 +74,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); submitButtonClick(); diff --git a/airbyte-webapp/cypress/pages/createConnectorPage.ts b/airbyte-webapp/cypress/pages/createConnectorPage.ts index 43bde7228d2..1c153ea7d93 100644 --- a/airbyte-webapp/cypress/pages/createConnectorPage.ts +++ b/airbyte-webapp/cypress/pages/createConnectorPage.ts @@ -9,10 +9,11 @@ const pokemonNameInput = "input[name='connectionConfiguration.pokemon_name']"; const schemaInput = "[data-testid='tag-input'] input"; const destinationPathInput = "input[name='connectionConfiguration.destination_path']"; const optionalFieldsButton = "button[data-testid='optional-fields']"; +const xminOption = "label[data-testid='radio-option.1']"; export const selectServiceType = (type: string) => { // Make sure alpha connectors are visible in the grid, since they are hidden by default - cy.contains("label", "Alpha").click(); + cy.get("#filter-support-level-community").check({ force: true }); cy.contains("button", type).click(); }; @@ -72,3 +73,7 @@ export const removeSchema = (value = "Remove public") => { export const openOptionalFields = () => { cy.get(optionalFieldsButton).click(); }; + +export const selectXmin = () => { + cy.get(xminOption).click(); +}; diff --git a/airbyte-webapp/jest.config.ts b/airbyte-webapp/jest.config.ts index 4063c727a08..9a07bbb9406 100644 --- a/airbyte-webapp/jest.config.ts +++ b/airbyte-webapp/jest.config.ts @@ -1,6 +1,6 @@ import type { Config } from "jest"; -const isInCI = process.env.CI; +const isInCI = process.env.DAGGER; const jestConfig: Config = { verbose: true, diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 5a22a46ac25..edec569468d 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,5 @@ { "name": "airbyte-webapp", - "version": "0.50.23", "private": true, "engines": { "node": "18.15.0", @@ -34,6 +33,7 @@ "license-check": "node ./scripts/license-check.js", "generate-client": "./scripts/load-declarative-schema.sh && orval", "validate-links": "ts-node --skip-project ./scripts/validate-links.ts", + "validate-lock": "node ./scripts/validate-lock-files.js", "preanalyze-lowcode": "TS_NODE_TRANSPILE_ONLY=true pnpm run generate-client", "analyze-lowcode": "ts-node --skip-project ./scripts/analyze-low-code-manifests.ts", "cypress:open": "cypress open --config-file cypress/cypress.config.ts", @@ -79,6 +79,7 @@ "@types/uuid": "^9.0.0", "anser": "^2.1.1", "any-base": "^1.1.0", + "byte-size": "^8.1.1", "classnames": "^2.3.1", "country-flag-icons": "^1.5.7", "date-fns": "^2.29.3", @@ -92,6 +93,7 @@ "json-schema": "^0.4.0", "launchdarkly-js-client-sdk": "^3.1.0", "lodash": "^4.17.21", + "markdown-to-jsx": "^7.3.2", "monaco-editor": "^0.34.1", "normalize.css": "^8.0.1", "oidc-client-ts": "^2.2.4", @@ -112,7 +114,6 @@ "react-rnd": "^10.4.1", "react-router-dom": "6.14.2", "react-select": "^5.4.0", - "react-slick": "^0.29.0", "react-use": "^17.4.0", "react-virtualized-auto-sizer": "^1.0.17", "react-widgets": "^4.6.1", @@ -120,6 +121,7 @@ "recharts": "^2.1.13", "rehype-slug": "^5.0.1", "rehype-urls": "^1.1.1", + "remark": "^14.0.3", "remark-directive": "^2.0.1", "remark-frontmatter": "^4.0.1", "remark-gfm": "^3.0.0", @@ -144,16 +146,17 @@ "@babel/preset-typescript": "^7.21.0", "@formatjs/icu-messageformat-parser": "^2.4.0", "@modyfi/vite-plugin-yaml": "^1.0.4", - "@storybook/addon-actions": "^7.0.2", - "@storybook/addon-docs": "^7.0.2", - "@storybook/addon-essentials": "^7.0.2", - "@storybook/addon-links": "^7.0.2", - "@storybook/react": "^7.0.2", - "@storybook/react-vite": "^7.0.2", - "@storybook/theming": "^7.0.2", + "@storybook/addon-actions": "^7.3.2", + "@storybook/addon-docs": "^7.3.2", + "@storybook/addon-essentials": "^7.3.2", + "@storybook/addon-links": "^7.3.2", + "@storybook/react": "^7.3.2", + "@storybook/react-vite": "^7.3.2", + "@storybook/theming": "^7.3.2", "@testing-library/jest-dom": "^6.0.0", "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.4.3", + "@types/byte-size": "^8.1.0", "@types/flat": "^5.0.2", "@types/jest": "^29.5.3", "@types/js-yaml": "^4.0.5", @@ -208,7 +211,8 @@ "prettier": "^2.8.4", "react-select-event": "^5.5.0", "start-server-and-test": "^2.0.0", - "storybook": "^7.0.2", + "storybook": "^7.3.2", + "storybook-dark-mode": "^3.0.1", "stylelint": "^14.9.1", "stylelint-config-css-modules": "^4.2.0", "stylelint-config-prettier-scss": "^0.0.1", diff --git a/airbyte-webapp/packages/eslint-plugin/index.js b/airbyte-webapp/packages/eslint-plugin/index.js index e874928d7d4..656b62d37c4 100644 --- a/airbyte-webapp/packages/eslint-plugin/index.js +++ b/airbyte-webapp/packages/eslint-plugin/index.js @@ -1,11 +1,13 @@ module.exports = { rules: { + "no-local-storage": require("./no-local-storage"), "no-hardcoded-connector-ids": require("./no-hardcoded-connector-ids"), }, configs: { recommended: { rules: { "@airbyte/no-hardcoded-connector-ids": "error", + "@airbyte/no-local-storage": "error", }, }, }, diff --git a/airbyte-webapp/packages/eslint-plugin/no-local-storage.js b/airbyte-webapp/packages/eslint-plugin/no-local-storage.js new file mode 100644 index 00000000000..191c6d890a7 --- /dev/null +++ b/airbyte-webapp/packages/eslint-plugin/no-local-storage.js @@ -0,0 +1,14 @@ +module.exports = { + create(context) { + return { + MemberExpression(node) { + if (node.object.name === "localStorage") { + context.report({ + node, + message: "Use the type-safe useLocalStorage hook instead of the global localStorage object", + }); + } + }, + }; + }, +}; diff --git a/airbyte-webapp/packages/vite-plugins/doc-middleware.ts b/airbyte-webapp/packages/vite-plugins/doc-middleware.ts index 6d809c34b10..936323312c0 100644 --- a/airbyte-webapp/packages/vite-plugins/doc-middleware.ts +++ b/airbyte-webapp/packages/vite-plugins/doc-middleware.ts @@ -19,10 +19,7 @@ const localDocMiddleware = (docsPath: string): Plugin => { express.static(`${docsPath}/sources/google-ads.md`) as Connect.NextHandleFunction ); // Server assets that can be used during. Related gradle task: :airbyte-webapp:copyDocAssets - server.middlewares.use( - "/docs/.gitbook", - express.static(`${docsPath}/docs/.gitbook`) as Connect.NextHandleFunction - ); + server.middlewares.use("/docs/.gitbook", express.static(`${docsPath}/../.gitbook`) as Connect.NextHandleFunction); }, }; }; diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index 14ee3a97a37..a2b5d51b6f7 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -96,6 +96,9 @@ dependencies: any-base: specifier: ^1.1.0 version: 1.1.0 + byte-size: + specifier: ^8.1.1 + version: 8.1.1 classnames: specifier: ^2.3.1 version: 2.3.2 @@ -135,6 +138,9 @@ dependencies: lodash: specifier: ^4.17.21 version: 4.17.21 + markdown-to-jsx: + specifier: ^7.3.2 + version: 7.3.2(react@18.2.0) monaco-editor: specifier: ^0.34.1 version: 0.34.1 @@ -195,9 +201,6 @@ dependencies: react-select: specifier: ^5.4.0 version: 5.7.0(@babel/core@7.21.3)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) - react-slick: - specifier: ^0.29.0 - version: 0.29.0(react-dom@18.2.0)(react@18.2.0) react-use: specifier: ^17.4.0 version: 17.4.0(react-dom@18.2.0)(react@18.2.0) @@ -219,6 +222,9 @@ dependencies: rehype-urls: specifier: ^1.1.1 version: 1.1.1 + remark: + specifier: ^14.0.3 + version: 14.0.3 remark-directive: specifier: ^2.0.1 version: 2.0.1 @@ -288,26 +294,26 @@ devDependencies: specifier: ^1.0.4 version: 1.0.4(rollup@2.79.1)(vite@4.2.0) '@storybook/addon-actions': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-docs': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-essentials': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-links': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0) '@storybook/react': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) '@storybook/react-vite': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2)(vite@4.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0)(rollup@2.79.1)(typescript@5.0.2)(vite@4.2.0) '@storybook/theming': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0) '@testing-library/jest-dom': specifier: ^6.0.0 version: 6.0.0(@types/jest@29.5.3)(jest@29.6.2) @@ -317,6 +323,9 @@ devDependencies: '@testing-library/user-event': specifier: ^14.4.3 version: 14.4.3(@testing-library/dom@9.3.1) + '@types/byte-size': + specifier: ^8.1.0 + version: 8.1.0 '@types/flat': specifier: ^5.0.2 version: 5.0.2 @@ -480,8 +489,11 @@ devDependencies: specifier: ^2.0.0 version: 2.0.0 storybook: - specifier: ^7.0.2 - version: 7.0.2 + specifier: ^7.3.2 + version: 7.3.2 + storybook-dark-mode: + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) stylelint: specifier: ^14.9.1 version: 14.16.1 @@ -538,6 +550,14 @@ packages: '@jridgewell/gen-mapping': 0.1.1 '@jridgewell/trace-mapping': 0.3.17 + /@ampproject/remapping@2.2.1: + resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.19 + dev: true + /@apidevtools/json-schema-ref-parser@9.0.6: resolution: {integrity: sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==} dependencies: @@ -576,8 +596,8 @@ packages: '@types/json-schema': 7.0.11 dev: true - /@aw-web-design/x-default-browser@1.4.88: - resolution: {integrity: sha512-AkEmF0wcwYC2QkhK703Y83fxWARttIWXDmQN8+cof8FmFZ5BRhnNXGymeb1S73bOCLfWjYELxtujL56idCN/XA==} + /@aw-web-design/x-default-browser@1.4.126: + resolution: {integrity: sha512-Xk1sIhyNC/esHGGVjL/niHLowM0csl/kFO5uawBy4IrWwy0o1G8LGt3jP6nmWGz+USxeeqbihAmp/oVZju6wug==} hasBin: true dependencies: default-browser-id: 3.0.0 @@ -601,8 +621,8 @@ packages: resolution: {integrity: sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg==} engines: {node: '>=6.9.0'} - /@babel/compat-data@7.21.4: - resolution: {integrity: sha512-/DYyDpeCfaVinT40FPGdkkb+lYSKvsVuMjDAG7jPOWWiM1ibOaB9CXJAlc4d1QpP/U2q2P9jbrSlClKSErd55g==} + /@babel/compat-data@7.22.9: + resolution: {integrity: sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ==} engines: {node: '>=6.9.0'} dev: true @@ -628,6 +648,29 @@ packages: transitivePeerDependencies: - supports-color + /@babel/core@7.22.11: + resolution: {integrity: sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@babel/code-frame': 7.22.10 + '@babel/generator': 7.22.10 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helpers': 7.22.11 + '@babel/parser': 7.22.11 + '@babel/template': 7.22.5 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + convert-source-map: 1.9.0 + debug: 4.3.4(supports-color@5.5.0) + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/eslint-parser@7.19.1(@babel/core@7.21.3)(eslint@8.36.0): resolution: {integrity: sha512-AqNf2QWt1rtu2/1rLswy6CDP7H9Oh3mMhk177Y67Rg8d7RD9WfOLLv8CGn6tisFvS2htm86yIe1yLF6I1UDaGQ==} engines: {node: ^10.13.0 || ^12.13.0 || >=14.0.0} @@ -671,7 +714,7 @@ packages: resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/helper-builder-binary-assignment-operator-visitor@7.18.9: @@ -682,6 +725,13 @@ packages: '@babel/types': 7.20.7 dev: true + /@babel/helper-builder-binary-assignment-operator-visitor@7.22.10: + resolution: {integrity: sha512-Av0qubwDQxC56DoUReVDeLfMEjYYSN1nZrTUrWkXd7hpU73ymRANkbuDm3yni9npkn+RXy9nNbEJZEzXr7xrfQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-compilation-targets@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==} engines: {node: '>=6.9.0'} @@ -695,18 +745,15 @@ packages: lru-cache: 5.1.1 semver: 6.3.0 - /@babel/helper-compilation-targets@7.21.4(@babel/core@7.21.3): - resolution: {integrity: sha512-Fa0tTuOXZ1iL8IeDFUWCzjZcn+sJGd9RZdH9esYVjEejGmzf+FFYQpMi/kZUk2kPy/q1H3/GPw7np8qar/stfg==} + /@babel/helper-compilation-targets@7.22.10: + resolution: {integrity: sha512-JMSwHD4J7SLod0idLq5PKgI+6g/hLD/iuWBq08ZX49xE14VpVEojJ5rHWptpirV2j020MvypRLAXAO50igCJ5Q==} engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 dependencies: - '@babel/compat-data': 7.21.4 - '@babel/core': 7.21.3 - '@babel/helper-validator-option': 7.21.0 - browserslist: 4.21.4 + '@babel/compat-data': 7.22.9 + '@babel/helper-validator-option': 7.22.5 + browserslist: 4.21.10 lru-cache: 5.1.1 - semver: 6.3.0 + semver: 6.3.1 dev: true /@babel/helper-create-class-features-plugin@7.21.0(@babel/core@7.21.3): @@ -728,6 +775,24 @@ packages: - supports-color dev: true + /@babel/helper-create-class-features-plugin@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-y1grdYL4WzmUDBRGK0pDbIoFd7UZKoDurDzWEoNMYoj1EL+foGRQNyPWDcC+YyegN5y1DUsFFmzjGijB3nSVAQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-member-expression-to-functions': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + semver: 6.3.1 + dev: true + /@babel/helper-create-regexp-features-plugin@7.20.5(@babel/core@7.21.3): resolution: {integrity: sha512-m68B1lkg3XDGX5yCvGO0kPx3v9WIYLnzjKfPcQiwntEQa5ZeRkPmo2X/ISJc8qxWGfwUr+kvZAeEzAwLec2r2w==} engines: {node: '>=6.9.0'} @@ -739,6 +804,18 @@ packages: regexpu-core: 5.2.2 dev: true + /@babel/helper-create-regexp-features-plugin@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-+svjVa/tFwsNSG4NEy1h85+HQ5imbT92Q5/bgtS7P0GTQlP8WuFdqsiABmQouhiFGyV66oGxZFpeYHza1rNsKw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + regexpu-core: 5.3.2 + semver: 6.3.1 + dev: true + /@babel/helper-define-polyfill-provider@0.3.3(@babel/core@7.21.3): resolution: {integrity: sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==} peerDependencies: @@ -755,10 +832,30 @@ packages: - supports-color dev: true + /@babel/helper-define-polyfill-provider@0.4.2(@babel/core@7.22.11): + resolution: {integrity: sha512-k0qnnOqHn5dK9pZpfD5XXZ9SojAITdCKRn2Lp6rnDGzIbaP0rHyMPk/4wsSxVBVz4RfN0q6VpXWP2pDGIoQ7hw==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + debug: 4.3.4(supports-color@5.5.0) + lodash.debounce: 4.0.8 + resolve: 1.22.4 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/helper-environment-visitor@7.18.9: resolution: {integrity: sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==} engines: {node: '>=6.9.0'} + /@babel/helper-environment-visitor@7.22.5: + resolution: {integrity: sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==} + engines: {node: '>=6.9.0'} + dev: true + /@babel/helper-explode-assignable-expression@7.18.6: resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} engines: {node: '>=6.9.0'} @@ -773,12 +870,27 @@ packages: '@babel/template': 7.20.7 '@babel/types': 7.21.3 + /@babel/helper-function-name@7.22.5: + resolution: {integrity: sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.5 + '@babel/types': 7.22.11 + dev: true + /@babel/helper-hoist-variables@7.18.6: resolution: {integrity: sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-hoist-variables@7.22.5: + resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-member-expression-to-functions@7.21.0: resolution: {integrity: sha512-Muu8cdZwNN6mRRNG6lAYErJ5X3bRevgYR2O8wN0yn7jJSnGDu6eG59RfT29JHxGUovyfrh6Pj0XzmR7drNVL3Q==} engines: {node: '>=6.9.0'} @@ -786,6 +898,13 @@ packages: '@babel/types': 7.21.4 dev: true + /@babel/helper-member-expression-to-functions@7.22.5: + resolution: {integrity: sha512-aBiH1NKMG0H2cGZqspNvsaBe6wNGjbJjuLy29aU+eDZjSbbN53BaxlpB02xm9v34pLTZ1nIQPFYn2qMZoa5BQQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-module-imports@7.18.6: resolution: {integrity: sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==} engines: {node: '>=6.9.0'} @@ -796,7 +915,7 @@ packages: resolution: {integrity: sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/helper-module-transforms@7.21.2: @@ -814,6 +933,34 @@ packages: transitivePeerDependencies: - supports-color + /@babel/helper-module-transforms@7.22.9(@babel/core@7.21.3): + resolution: {integrity: sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + + /@babel/helper-module-transforms@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + /@babel/helper-optimise-call-expression@7.18.6: resolution: {integrity: sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==} engines: {node: '>=6.9.0'} @@ -821,6 +968,13 @@ packages: '@babel/types': 7.21.3 dev: true + /@babel/helper-optimise-call-expression@7.22.5: + resolution: {integrity: sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-plugin-utils@7.20.2: resolution: {integrity: sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ==} engines: {node: '>=6.9.0'} @@ -845,6 +999,18 @@ packages: - supports-color dev: true + /@babel/helper-remap-async-to-generator@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-8WWC4oR4Px+tr+Fp0X3RHDVfINGpF3ad1HIbrc8A77epiR6eMMc6jsgozkzT2uDiOOdoS9cLIQ+XD2XvI2WSmQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-wrap-function': 7.22.10 + dev: true + /@babel/helper-replace-supers@7.20.7: resolution: {integrity: sha512-vujDMtB6LVfNW13jhlCrp48QNslK6JXi7lQG736HVbHz/mbf4Dc7tIRh1Xf5C0rF7BP8iiSxGMCmY6Ci1ven3A==} engines: {node: '>=6.9.0'} @@ -859,12 +1025,31 @@ packages: - supports-color dev: true + /@babel/helper-replace-supers@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-LJIKvvpgPOPUThdYqcX6IXRuIcTkcAub0IaDRGCZH0p5GPUp7PhRU9QVgFcDDd51BaPkk77ZjqFwh6DZTAEmGg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-member-expression-to-functions': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + dev: true + /@babel/helper-simple-access@7.20.2: resolution: {integrity: sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-simple-access@7.22.5: + resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-skip-transparent-expression-wrappers@7.20.0: resolution: {integrity: sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==} engines: {node: '>=6.9.0'} @@ -872,12 +1057,26 @@ packages: '@babel/types': 7.21.3 dev: true + /@babel/helper-skip-transparent-expression-wrappers@7.22.5: + resolution: {integrity: sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-split-export-declaration@7.18.6: resolution: {integrity: sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-split-export-declaration@7.22.6: + resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-string-parser@7.19.4: resolution: {integrity: sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==} engines: {node: '>=6.9.0'} @@ -905,6 +1104,11 @@ packages: resolution: {integrity: sha512-rmL/B8/f0mKS2baE9ZpyTcTavvEuWhTTW8amjzXNvYG4AwBsqTLikfXsEofsJEfKHf+HQVQbFOHy6o+4cnC/fQ==} engines: {node: '>=6.9.0'} + /@babel/helper-validator-option@7.22.5: + resolution: {integrity: sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==} + engines: {node: '>=6.9.0'} + dev: true + /@babel/helper-wrap-function@7.20.5: resolution: {integrity: sha512-bYMxIWK5mh+TgXGVqAtnu5Yn1un+v8DDZtqyzKRLUzrh70Eal2O3aZ7aPYiMADO4uKlkzOiRiZ6GX5q3qxvW9Q==} engines: {node: '>=6.9.0'} @@ -917,6 +1121,15 @@ packages: - supports-color dev: true + /@babel/helper-wrap-function@7.22.10: + resolution: {integrity: sha512-OnMhjWjuGYtdoO3FmsEFWvBStBAe2QOgwOLsLNDjN+aaiMD8InJk1/O3HSD8lkqTjCgg5YI34Tz15KNNA3p+nQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-function-name': 7.22.5 + '@babel/template': 7.22.5 + '@babel/types': 7.22.11 + dev: true + /@babel/helpers@7.21.0: resolution: {integrity: sha512-XXve0CBtOW0pd7MRzzmoyuSj0e3SEzj8pgyFxnTT1NJZL38BD1MK7yYrm8yefRPIDvNNe14xR4FdbHwpInD4rA==} engines: {node: '>=6.9.0'} @@ -927,6 +1140,17 @@ packages: transitivePeerDependencies: - supports-color + /@babel/helpers@7.22.11: + resolution: {integrity: sha512-vyOXC8PBWaGc5h7GMsNx68OH33cypkEDJCHvYVVgVbbxJDROYVtexSk0gK5iCF1xNjRIN2s8ai7hwkWDq5szWg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.5 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/highlight@7.18.6: resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} engines: {node: '>=6.9.0'} @@ -967,6 +1191,14 @@ packages: '@babel/types': 7.22.10 dev: true + /@babel/parser@7.22.11: + resolution: {integrity: sha512-R5zb8eJIBPJriQtbH/htEQy4k7E2dHWlD2Y2VT07JCzwYZHBxV5ZYtM0UhXSNMT74LyxuM+b1jdL7pSesXbC/g==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==} engines: {node: '>=6.9.0'} @@ -977,6 +1209,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-NP1M5Rf+u2Gw9qfSO4ihjcTGW5zXTi36ITLd4/EoAcEhIZ0yjMqmftDNl3QC19CX7olhrjpyU454g/2W7X0jvQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-sbr9+wNE5aXMBBFBICk01tt7sBf2Oc9ikRFEcem/ZORup9IMUdNhW7/wVLEbbtlWOsEubJet46mHAL2C8+2jKQ==} engines: {node: '>=6.9.0'} @@ -989,6 +1231,18 @@ packages: '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) dev: true + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-31Bb65aZaUwqCbWMnZPduIZxCBngHFlzyN6Dq6KAJjtx+lx6ohKHubc61OomYi7XwVD4Ol0XCVz4h+pYFR048g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.13.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-transform-optional-chaining': 7.22.12(@babel/core@7.22.11) + dev: true + /@babel/plugin-proposal-async-generator-functions@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==} engines: {node: '>=6.9.0'} @@ -1031,20 +1285,6 @@ packages: - supports-color dev: true - /@babel/plugin-proposal-class-static-block@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.12.0 - dependencies: - '@babel/core': 7.21.3 - '@babel/helper-create-class-features-plugin': 7.21.0(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.21.3) - transitivePeerDependencies: - - supports-color - dev: true - /@babel/plugin-proposal-decorators@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-JB45hbUweYpwAGjkiM7uCyXMENH2lG+9r3G2E+ttc2PRXAoEkpfd/KW5jDg4j8RS6tLtTG1jZi9LbHZVSfs1/A==} engines: {node: '>=6.9.0'} @@ -1171,8 +1411,8 @@ packages: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.21.3) dev: true @@ -1204,19 +1444,13 @@ packages: - supports-color dev: true - /@babel/plugin-proposal-private-property-in-object@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-ha4zfehbJjc5MmXBlHec1igel5TJXXLDDRbuJ4+XT2TJcyD9/V1919BA8gMvsdHcNMBy4WBUBiRb3nw/EQUtBw==} + /@babel/plugin-proposal-private-property-in-object@7.21.0-placeholder-for-preset-env.2(@babel/core@7.22.11): + resolution: {integrity: sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-annotate-as-pure': 7.18.6 - '@babel/helper-create-class-features-plugin': 7.21.0(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.21.3) - transitivePeerDependencies: - - supports-color + '@babel/core': 7.22.11 dev: true /@babel/plugin-proposal-unicode-property-regex@7.18.6(@babel/core@7.21.3): @@ -1239,6 +1473,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.22.11): + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==} peerDependencies: @@ -1257,6 +1500,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.22.11): + resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} engines: {node: '>=6.9.0'} @@ -1267,6 +1519,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-decorators@7.19.0(@babel/core@7.21.3): resolution: {integrity: sha512-xaBZUEDntt4faL1yN8oIFlhfXeQAWJW7CLKYsHTUqriCUbj8xOra8bfxxKGi/UwExPFBuPdH4XfHc9rGQhrVkQ==} engines: {node: '>=6.9.0'} @@ -1286,6 +1548,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-dynamic-import@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} peerDependencies: @@ -1295,6 +1566,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-flow@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==} engines: {node: '>=6.9.0'} @@ -1325,6 +1605,26 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-import-assertions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-rdV97N7KqsRzeNGoWUOK6yUsWarLjE5Su/Snk9IYPU9CwkWHs4t+rTGOvffTR8XGkJMTAdLfO0xVnXm8wugIJg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-syntax-import-attributes@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-KwvoWDeNKPETmozyFE0P2rOLqh39EoQHNjqizrI5B8Vt0ZNS7M56s7dAiAqbYfiAYOuIzIh96z3iR2ktgu3tEg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.21.3): resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} peerDependencies: @@ -1334,6 +1634,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} peerDependencies: @@ -1343,6 +1652,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-jsx@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q==} engines: {node: '>=6.9.0'} @@ -1371,6 +1689,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} peerDependencies: @@ -1380,17 +1707,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.21.3): - resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 + '@babel/core': 7.22.11 '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.21.3): - resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.21.3): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: @@ -1398,6 +1725,33 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.21.3): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} peerDependencies: @@ -1407,6 +1761,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} peerDependencies: @@ -1416,6 +1779,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} engines: {node: '>=6.9.0'} @@ -1426,6 +1798,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} engines: {node: '>=6.9.0'} @@ -1436,6 +1818,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-typescript@7.20.0(@babel/core@7.21.3): resolution: {integrity: sha512-rd9TkG+u1CExzS4SM1BlMEhMXwFLKVjOAFFCDx9PbX5ycJWDoWMcwdJH9RhkPu1dOgn5TrxLot/Gx6lWFuAUNQ==} engines: {node: '>=6.9.0'} @@ -1456,6 +1848,17 @@ packages: '@babel/helper-plugin-utils': 7.22.5 dev: true + /@babel/plugin-syntax-unicode-sets-regex@7.18.6(@babel/core@7.22.11): + resolution: {integrity: sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-arrow-functions@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-3poA5E7dzDomxj9WXWwuD6A5F3kc7VXwIJO+E+J8qtDtS+pXPAhrgEyh+9GBwBgPq1Z+bB+/JD60lp5jsN7JPQ==} engines: {node: '>=6.9.0'} @@ -1466,6 +1869,29 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-arrow-functions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-26lTNXoVRdAnsaDXPpvCNUq+OVWEVC6bx7Vvz9rC53F2bagUWW4u4ii2+h8Fejfh7RYqPxn+libeFBBck9muEw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-async-generator-functions@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-0pAlmeRJn6wU84zzZsEOx1JV1Jf8fqO9ok7wofIJwUnplYo247dcd24P+cMJht7ts9xkzdtB0EPHmOb7F+KzXw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.9(@babel/core@7.22.11) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-async-to-generator@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-Uo5gwHPT9vgnSXQxqGtpdufUiWp96gk7yiP4Mp5bm1QMkEmLXBO7PAGYbKoJ6DhAwiNkcHFBol/x5zZZkL/t0Q==} engines: {node: '>=6.9.0'} @@ -1480,6 +1906,18 @@ packages: - supports-color dev: true + /@babel/plugin-transform-async-to-generator@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-b1A8D8ZzE/VhNDoV1MSJTnpKkCG5bJo+19R4o4oy03zM7ws8yEMK755j61Dc3EyvdysbqH5BOOTquJ7ZX9C6vQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.9(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-block-scoped-functions@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==} engines: {node: '>=6.9.0'} @@ -1490,6 +1928,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-block-scoped-functions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-tdXZ2UdknEKQWKJP1KMNmuF5Lx3MymtMN/pvA+p/VEkhK8jVcQ1fzSy8KM9qRYhAf2/lV33hoMPKI/xaI9sADA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-block-scoping@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-tA4N427a7fjf1P0/2I4ScsHGc5jcHPbb30xMbaTke2gxDuWpUfXDuX1FEymJwKk4tuGUvGcejAR6HdZVqmmPyw==} engines: {node: '>=6.9.0'} @@ -1500,14 +1948,37 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-block-scoping@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-Mdrbunoh9SxwFZapeHVrwFmri16+oYotcZysSzhNIVDwIAb1UV+kvnxULSYq9J3/q5MDG+4X6w8QVgD1zhBXNQ==} + /@babel/plugin-transform-block-scoping@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-1+kVpGAOOI1Albt6Vse7c8pHzcZQdQKW+wJH+g8mCaszOdDVwRXa/slHPqIw+oJAJANTKDMuM2cBdV0Dg618Vg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-class-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-nDkQ0NfkOhPTq8YCLiWNxp1+f9fCobEjCb0n8WdbNUBc4IB5V7P1QnX9IjpSoquKrXF5SKojHleVNs2vGeHCHQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-class-static-block@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-GMM8gGmqI7guS/llMFk1bJDkKfn3v3C4KHK9Yg1ey5qcHcOlKb0QvcMrgzvxo+T03/4szNh5lghY+fEC98Kq9g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.22.11) dev: true /@babel/plugin-transform-classes@7.20.7(@babel/core@7.21.3): @@ -1530,24 +2001,22 @@ packages: - supports-color dev: true - /@babel/plugin-transform-classes@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-RZhbYTCEUAe6ntPehC4hlslPWosNHDox+vAs4On/mCLRLfoDVHf6hVEd7kuxr1RnHwJmxFfUM3cZiZRmPxJPXQ==} + /@babel/plugin-transform-classes@7.22.6(@babel/core@7.22.11): + resolution: {integrity: sha512-58EgM6nuPNG6Py4Z3zSuu0xWu2VfodiMi72Jt5Kj2FECmaYk1RrTXA45z6KBFsu9tRgwQDwIiY4FXTt+YsSFAQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-annotate-as-pure': 7.18.6 - '@babel/helper-compilation-targets': 7.21.4(@babel/core@7.21.3) - '@babel/helper-environment-visitor': 7.18.9 - '@babel/helper-function-name': 7.21.0 - '@babel/helper-optimise-call-expression': 7.18.6 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-replace-supers': 7.20.7 - '@babel/helper-split-export-declaration': 7.18.6 + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + '@babel/helper-split-export-declaration': 7.22.6 globals: 11.12.0 - transitivePeerDependencies: - - supports-color dev: true /@babel/plugin-transform-computed-properties@7.20.7(@babel/core@7.21.3): @@ -1561,6 +2030,17 @@ packages: '@babel/template': 7.20.7 dev: true + /@babel/plugin-transform-computed-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-4GHWBgRf0krxPX+AaPtgBAlTgTeZmqDynokHOX7aqqAB4tHs3U2Y02zH6ETFdLZGcg9UQSD1WCmkVrE9ErHeOg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/template': 7.22.5 + dev: true + /@babel/plugin-transform-destructuring@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-Xwg403sRrZb81IVB79ZPqNQME23yhugYVqgTxAhT99h485F4f+GMELFhhOsscDUB7HCswepKeCKLn/GZvUKoBA==} engines: {node: '>=6.9.0'} @@ -1571,14 +2051,14 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-destructuring@7.21.3(@babel/core@7.21.3): - resolution: {integrity: sha512-bp6hwMFzuiE4HqYEyoGJ/V2LeIWn+hLVKc4pnj++E5XQptwhtcGmSayM029d/j2X1bPKGTlsyPwAubuU22KhMA==} + /@babel/plugin-transform-destructuring@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-dPJrL0VOyxqLM9sritNbMSGx/teueHF/htMKrPT7DNxccXxRDPYqlgPFFdr8u+F+qUZOkZoXue/6rL5O5GduEw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 dev: true /@babel/plugin-transform-dotall-regex@7.18.6(@babel/core@7.21.3): @@ -1592,6 +2072,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-dotall-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5/Yk9QxCQCl+sOIB1WelKnVRxTJDSAIxtJLL2/pqL14ZVlbH0fUQUZa/T5/UnQtBNgghR7mfB8ERBKyKPCi7Vw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-duplicate-keys@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw==} engines: {node: '>=6.9.0'} @@ -1602,6 +2093,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-duplicate-keys@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-dEnYD+9BBgld5VBXHnF/DbYGp3fqGMsyxKbtD1mDyIA7AkTSpKXFhCVuj/oQVOoALfBs77DudA0BE4d5mcpmqw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-dynamic-import@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-g/21plo58sfteWjaO0ZNVb+uEOkJNjAaHhbejrnBmu011l/eNDScmkbjCC3l4FKb10ViaGU4aOkFznSu2zRHgA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-exponentiation-operator@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==} engines: {node: '>=6.9.0'} @@ -1613,6 +2125,28 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-exponentiation-operator@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-vIpJFNM/FjZ4rh1myqIya9jXwrwwgFRHPjT3DkUA9ZLHuzox8jiXkOLvwm1H+PQIP3CqfC++WPKeuDi0Sjdj1g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-builder-binary-assignment-operator-visitor': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-export-namespace-from@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-xa7aad7q7OiT8oNZ1mU7NrISjlSkVdMbNxn9IuLZyL9AJEhs1Apba3I+u5riX1dIkdptP5EKDG5XDPByWxtehw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-flow-strip-types@7.19.0(@babel/core@7.21.3): resolution: {integrity: sha512-sgeMlNaQVbCSpgLSKP4ZZKfsJVnFnNQlUSk6gPYzR/q7tzCgQF2t8RBKAP6cKJeZdveei7Q7Jm527xepI8lNLg==} engines: {node: '>=6.9.0'} @@ -1624,6 +2158,17 @@ packages: '@babel/plugin-syntax-flow': 7.18.6(@babel/core@7.21.3) dev: true + /@babel/plugin-transform-flow-strip-types@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-tujNbZdxdG0/54g/oua8ISToaXTFBf8EnSb5PgQSciIXWOWKX3S4+JR7ZE9ol8FZwf9kxitzkGQ+QWeov/mCiA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-flow': 7.22.5(@babel/core@7.21.3) + dev: true + /@babel/plugin-transform-for-of@7.18.8(@babel/core@7.21.3): resolution: {integrity: sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==} engines: {node: '>=6.9.0'} @@ -1634,14 +2179,14 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-for-of@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-LlUYlydgDkKpIY7mcBWvyPPmMcOphEyYA27Ef4xpbh1IiDNLr0kZsos2nf92vz3IccvJI25QUwp86Eo5s6HmBQ==} + /@babel/plugin-transform-for-of@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-3kxQjX1dU9uudwSshyLeEipvrLjBCVthCgeTp6CzE/9JYrlAIaeekVxRpCWsDDfYTfRZRoCeZatCQvwo+wvK8A==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 dev: true /@babel/plugin-transform-function-name@7.18.9(@babel/core@7.21.3): @@ -1656,6 +2201,29 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-function-name@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-UIzQNMS0p0HHiQm3oelztj+ECwFnj+ZRV4KnguvlsD2of1whUeM6o7wGNj6oLwcDoAXQ8gEqfgC24D+VdIcevg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-json-strings@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-CxT5tCqpA9/jXFlme9xIBCc5RPtdDq3JpkkhgHQqtDdiTnTI0jtZ0QzXhr5DILeYifDPp2wvY2ad+7+hLMW5Pw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-literals@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==} engines: {node: '>=6.9.0'} @@ -1666,6 +2234,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-fTLj4D79M+mepcw3dgFBTIDYpbcB9Sm0bpm4ppXPaO+U+PKFFyV9MGRvS0gvGw62sd10kT5lRMKXAADb9pWy8g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-logical-assignment-operators@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-qQwRTP4+6xFCDV5k7gZBF3C31K34ut0tbEcTKxlX/0KXxm9GLcO14p570aWxFvVzx6QAfPgq7gaeIHXJC8LswQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-member-expression-literals@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==} engines: {node: '>=6.9.0'} @@ -1676,6 +2265,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-member-expression-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-RZEdkNtzzYCFl9SE9ATaUMTj2hqMb4StarOJLrZRbqqU4HSBE7UlBw9WBWQiDzrJZJdUWiMTVDI6Gv/8DPvfew==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-modules-amd@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-NuzCt5IIYOW0O30UvqktzHYR2ud5bOWbY0yaxWZ6G+aFzOMJvrs5YHNikrbdaT15+KNO31nPOy5Fim3ku6Zb5g==} engines: {node: '>=6.9.0'} @@ -1689,6 +2288,17 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-amd@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-R+PTfLTcYEmb1+kK7FNkhQ1gP4KgjpSO6HfH9+f8/yfp2Nt3ggBjiVpRwmwTlfqZLafYKJACy36yDXlEmI9HjQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-modules-commonjs@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-S8e1f7WQ7cimJQ51JkAaDrEtohVEitXjgCGAS2N8S31Y42E+kWwfSz83LYz57QdBm7q9diARVqanIaH2oVgQnw==} engines: {node: '>=6.9.0'} @@ -1703,18 +2313,28 @@ packages: - supports-color dev: true - /@babel/plugin-transform-modules-commonjs@7.21.2(@babel/core@7.21.3): - resolution: {integrity: sha512-Cln+Yy04Gxua7iPdj6nOV96smLGjpElir5YwzF0LBPKoPlLDNJePNlrGGaybAJkd0zKRnOVXOgizSqPYMNYkzA==} + /@babel/plugin-transform-modules-commonjs@7.22.11(@babel/core@7.21.3): + resolution: {integrity: sha512-o2+bg7GDS60cJMgz9jWqRUsWkMzLCxp+jFDeDUT5sjRlAxcJWZ2ylNdI7QQ2+CH5hWu7OnN+Cv3htt7AkSf96g==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-module-transforms': 7.21.2 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-simple-access': 7.20.2 - transitivePeerDependencies: - - supports-color + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.21.3) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + dev: true + + /@babel/plugin-transform-modules-commonjs@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-o2+bg7GDS60cJMgz9jWqRUsWkMzLCxp+jFDeDUT5sjRlAxcJWZ2ylNdI7QQ2+CH5hWu7OnN+Cv3htt7AkSf96g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 dev: true /@babel/plugin-transform-modules-systemjs@7.20.11(@babel/core@7.21.3): @@ -1732,6 +2352,19 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-systemjs@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-rIqHmHoMEOhI3VkVf5jQ15l539KrwhzqcBO6wdCNWPWc/JWt9ILNYNUssbRpeq0qWns8svuw8LnMNCvWBIJ8wA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + /@babel/plugin-transform-modules-umd@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==} engines: {node: '>=6.9.0'} @@ -1745,6 +2378,17 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-umd@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-+S6kzefN/E1vkSsKx8kmQuqeQsvCKCd1fraCM7zXm4SFoggI099Tr4G8U81+5gtMdUeMQ4ipdQffbKLX0/7dBQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-named-capturing-groups-regex@7.20.5(@babel/core@7.21.3): resolution: {integrity: sha512-mOW4tTzi5iTLnw+78iEq3gr8Aoq4WNRGpmSlrogqaiCBoR1HFhpU4JkpQFOHfeYx3ReVIFWOQJS4aZBRvuZ6mA==} engines: {node: '>=6.9.0'} @@ -1756,6 +2400,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-named-capturing-groups-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-new-target@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==} engines: {node: '>=6.9.0'} @@ -1766,19 +2421,99 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-object-super@7.18.6(@babel/core@7.21.3): - resolution: {integrity: sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==} + /@babel/plugin-transform-new-target@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-AsF7K0Fx/cNKVyk3a+DW0JLo+Ua598/NxMRvxDnkpCIGFh43+h/v2xyhRUYf6oD8gE4QtL83C7zZVghMjHd+iw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-nullish-coalescing-operator@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-YZWOw4HxXrotb5xsjMJUDlLgcDXSfO9eCmdl1bgW4+/lAGdkjaEvOnQ4p5WKKdUgSzO39dgPl0pTnfxm0OAXcg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-numeric-separator@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-3dzU4QGPsILdJbASKhF/V2TVP+gJya1PsueQCxIPCEcerqF21oEcrob4mzjsp2Py/1nLfF5m+xYNMDpmA8vffg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-object-rest-spread@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-nX8cPFa6+UmbepISvlf5jhQyaC7ASs/7UxHmMkuJ/k5xSHvDPPaibMo+v3TXwU/Pjqhep/nFNpd3zn4YR59pnw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-transform-parameters': 7.22.5(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-object-super@7.18.6(@babel/core@7.21.3): + resolution: {integrity: sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.20.2 '@babel/helper-replace-supers': 7.20.7 transitivePeerDependencies: - supports-color dev: true + /@babel/plugin-transform-object-super@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-klXqyaT9trSjIUrcsYIfETAzmOEZL3cBYqOYLJxBHfMFFggmXOv+NYSX/Jbs9mzMVESw/WycLFPRx8ba/b2Ipw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-optional-catch-binding@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-rli0WxesXUeCJnMYhzAglEjLWVDF6ahb45HuprcmQuLidBJFWjNnOzssk2kuc6e33FlLaiZhG/kUIzUMWdBKaQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-optional-chaining@7.22.12(@babel/core@7.22.11): + resolution: {integrity: sha512-7XXCVqZtyFWqjDsYDY4T45w4mlx1rf7aOgkc/Ww76xkgBiOlmjPkx36PBLHa1k1rwWvVgYMPsbuVnIamx2ZQJw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-parameters@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-WiWBIkeHKVOSYPO0pWkxGPfKeWrCJyD3NJ53+Lrp/QMSZbsVPovrVl2aWZ19D/LTVnaDv5Ap7GJ/B2CTOZdrfA==} engines: {node: '>=6.9.0'} @@ -1789,14 +2524,38 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-parameters@7.21.3(@babel/core@7.21.3): - resolution: {integrity: sha512-Wxc+TvppQG9xWFYatvCGPvZ6+SIUxQ2ZdiBP+PHYMIjnPXD+uThCshaz4NZOnODAtBjjcVQQ/3OKs9LW28purQ==} + /@babel/plugin-transform-parameters@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-AVkFUBurORBREOmHRKo06FjHYgjrabpdqRSwq6+C7R5iTCZOsM4QbcB27St0a4U6fffyAOqh3s/qEfybAhfivg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-private-methods@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-PPjh4gyrQnGe97JTalgRGMuU4icsZFnWkzicB/fUtzlKUqvsWBKEpPPfr5a2JiyirZkHxnAqkQMO5Z5B2kK3fA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-private-property-in-object@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-sSCbqZDBKHetvjSwpyWzhuHkmW5RummxJBVbYLkGkaiTOWGxml7SXt0iWa03bzxFIx7wOj3g/ILRd0RcJKBeSQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.22.11) dev: true /@babel/plugin-transform-property-literals@7.18.6(@babel/core@7.21.3): @@ -1809,6 +2568,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-property-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-TiOArgddK3mK/x1Qwf5hay2pxI6wCZnvQqrFSqbtg1GLl2JcNMitVH/YnqjP+M31pLUeTfzY1HAXFDnUBV30rQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-react-display-name@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-TV4sQ+T013n61uMoygyMRm+xf04Bd5oqFpv2jAEQwSZ8NwQA7zeRPg1LMVg2PWi3zWBz+CLKD+v5bcpZ/BS0aA==} engines: {node: '>=6.9.0'} @@ -1874,7 +2643,7 @@ packages: '@babel/helper-module-imports': 7.22.5 '@babel/helper-plugin-utils': 7.22.5 '@babel/plugin-syntax-jsx': 7.22.5(@babel/core@7.21.3) - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/plugin-transform-react-pure-annotations@7.18.6(@babel/core@7.21.3): @@ -1899,6 +2668,17 @@ packages: regenerator-transform: 0.15.1 dev: true + /@babel/plugin-transform-regenerator@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-F28b1mDt8KcT5bUyJc/U9nwzw6cV+UmTeRlXYIl2TNqMMJif0Jeey9/RQ3C4NOd2zp0/TRsDns9ttj2L523rsw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + regenerator-transform: 0.15.2 + dev: true + /@babel/plugin-transform-reserved-words@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==} engines: {node: '>=6.9.0'} @@ -1909,6 +2689,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-reserved-words@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-DTtGKFRQUDm8svigJzZHzb/2xatPc6TzNvAIJ5GqOKDsGFYgAskjRulbR/vGsPKq3OPqtexnz327qYpP57RFyA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-runtime@7.19.6(@babel/core@7.21.3): resolution: {integrity: sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw==} engines: {node: '>=6.9.0'} @@ -1936,6 +2726,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-shorthand-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-vM4fq9IXHscXVKzDv5itkO1X52SmdFBFcMIBZ2FRn2nqVYqw6dBexUgMvAjHW+KXpPPViD/Yo3GrDEBaRC0QYA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-spread@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-ewBbHQ+1U/VnH1fxltbJqDeWBU1oNLG8Dj11uIv3xVf7nrQu0bPGe5Rf716r7K5Qz+SqtAOVswoVunoiBtGhxw==} engines: {node: '>=6.9.0'} @@ -1947,6 +2747,17 @@ packages: '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 dev: true + /@babel/plugin-transform-spread@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5ZzDQIGyvN4w8+dMmpohL6MBo+l2G7tfC/O2Dg7/hjpgeWvUx8FzfeOKxGog9IimPa4YekaQ9PlDqTLOljkcxg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: true + /@babel/plugin-transform-sticky-regex@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==} engines: {node: '>=6.9.0'} @@ -1957,6 +2768,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-sticky-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-zf7LuNpHG0iEeiyCNwX4j3gDg1jgt1k3ZdXBKbZSoA3BbGQGvMiSvfbZRR3Dr3aeJe3ooWFZxOOG3IRStYp2Bw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-template-literals@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==} engines: {node: '>=6.9.0'} @@ -1967,6 +2788,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-template-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5ciOehRNf+EyUeewo8NkbQiUs4d6ZxiHo6BcBcnFlgiJfu16q0bQUw9Jvo0b0gBKFG1SMhDSjeKXSYuJLeFSMA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-typeof-symbol@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw==} engines: {node: '>=6.9.0'} @@ -1977,6 +2808,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-typeof-symbol@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-bYkI5lMzL4kPii4HHEEChkD0rkc+nvnlR6+o/qdqR6zrm0Sv/nodmyLhlq2DO0YKLUNd2VePmPRjJXSBh9OIdA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-typescript@7.21.3(@babel/core@7.21.3): resolution: {integrity: sha512-RQxPz6Iqt8T0uw/WsJNReuBpWpBqs/n7mNo18sKLoTbMp+UrEekhH+pKSVC7gWz+DNjo9gryfV8YzCiT45RgMw==} engines: {node: '>=6.9.0'} @@ -2002,6 +2843,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-unicode-escapes@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-lRfaRKGZCBqDlRU3UIFovdp9c9mEvlylmpod0/OatICsSfuQ9YFthRo1tpTkGsklEefZdqlEFdY4A2dwTb6ohg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-unicode-property-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-HCCIb+CbJIAE6sXn5CjFQXMwkCClcOfPCzTlilJ8cUatfzwHlWQkbtV0zD338u9dZskwvuOYTuuaMaA8J5EI5A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-unicode-regex@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==} engines: {node: '>=6.9.0'} @@ -2013,6 +2875,28 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-unicode-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-028laaOKptN5vHJf9/Arr/HiJekMd41hOEZYvNsrsXqJ7YPYuX2bQxh31fkZzGmq3YqHRJzYFFAVYvKfMPKqyg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-unicode-sets-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-lhMfi4FC15j13eKrh3DnYHjpGj6UKQHtNKTbtc1igvAhRy4+kLhV07OpLcsN0VgDEw/MjAvJO4BdMJsHwMhzCg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/preset-env@7.20.2(@babel/core@7.21.3): resolution: {integrity: sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==} engines: {node: '>=6.9.0'} @@ -2099,102 +2983,107 @@ packages: - supports-color dev: true - /@babel/preset-env@7.21.4(@babel/core@7.21.3): - resolution: {integrity: sha512-2W57zHs2yDLm6GD5ZpvNn71lZ0B/iypSdIeq25OurDKji6AdzV07qp4s3n1/x5BqtiGaTrPN3nerlSCaC5qNTw==} + /@babel/preset-env@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-riHpLb1drNkpLlocmSyEg4oYJIQFeXAK/d7rI6mbD0XsvoTOOweXDmQPG/ErxsEhWk3rl3Q/3F6RFQlVFS8m0A==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/compat-data': 7.21.4 - '@babel/core': 7.21.3 - '@babel/helper-compilation-targets': 7.21.4(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-validator-option': 7.21.0 - '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-async-generator-functions': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-class-static-block': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-dynamic-import': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-export-namespace-from': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-proposal-json-strings': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-logical-assignment-operators': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-numeric-separator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-object-rest-spread': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-catch-binding': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-private-methods': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-private-property-in-object': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-unicode-property-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.21.3) - '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.21.3) - '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-import-assertions': 7.20.0(@babel/core@7.21.3) - '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.21.3) - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.21.3) - '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-transform-arrow-functions': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-async-to-generator': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-block-scoped-functions': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-block-scoping': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-classes': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-computed-properties': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-destructuring': 7.21.3(@babel/core@7.21.3) - '@babel/plugin-transform-dotall-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-duplicate-keys': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-exponentiation-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-for-of': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-function-name': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-literals': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-member-expression-literals': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-modules-amd': 7.20.11(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.21.2(@babel/core@7.21.3) - '@babel/plugin-transform-modules-systemjs': 7.20.11(@babel/core@7.21.3) - '@babel/plugin-transform-modules-umd': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-named-capturing-groups-regex': 7.20.5(@babel/core@7.21.3) - '@babel/plugin-transform-new-target': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-object-super': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-parameters': 7.21.3(@babel/core@7.21.3) - '@babel/plugin-transform-property-literals': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-regenerator': 7.20.5(@babel/core@7.21.3) - '@babel/plugin-transform-reserved-words': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-shorthand-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-spread': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-sticky-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-template-literals': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-typeof-symbol': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-unicode-escapes': 7.18.10(@babel/core@7.21.3) - '@babel/plugin-transform-unicode-regex': 7.18.6(@babel/core@7.21.3) - '@babel/preset-modules': 0.1.5(@babel/core@7.21.3) - '@babel/types': 7.21.4 - babel-plugin-polyfill-corejs2: 0.3.3(@babel/core@7.21.3) - babel-plugin-polyfill-corejs3: 0.6.0(@babel/core@7.21.3) - babel-plugin-polyfill-regenerator: 0.4.1(@babel/core@7.21.3) - core-js-compat: 3.27.1 - semver: 6.3.0 + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.22.5 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-proposal-private-property-in-object': 7.21.0-placeholder-for-preset-env.2(@babel/core@7.22.11) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.22.11) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.22.11) + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-import-assertions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-syntax-import-attributes': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-unicode-sets-regex': 7.18.6(@babel/core@7.22.11) + '@babel/plugin-transform-arrow-functions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-async-generator-functions': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-async-to-generator': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-block-scoped-functions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-block-scoping': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-class-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-class-static-block': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-classes': 7.22.6(@babel/core@7.22.11) + '@babel/plugin-transform-computed-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-destructuring': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-dotall-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-duplicate-keys': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-dynamic-import': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-exponentiation-operator': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-export-namespace-from': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-for-of': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-function-name': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-json-strings': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-logical-assignment-operators': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-member-expression-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-modules-amd': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-modules-commonjs': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-modules-systemjs': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-modules-umd': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-named-capturing-groups-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-new-target': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-nullish-coalescing-operator': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-numeric-separator': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-object-rest-spread': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-object-super': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-optional-catch-binding': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-optional-chaining': 7.22.12(@babel/core@7.22.11) + '@babel/plugin-transform-parameters': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-private-methods': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-private-property-in-object': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-property-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-regenerator': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-reserved-words': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-shorthand-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-spread': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-sticky-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-template-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-typeof-symbol': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-escapes': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-property-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-sets-regex': 7.22.5(@babel/core@7.22.11) + '@babel/preset-modules': 0.1.6-no-external-plugins(@babel/core@7.22.11) + '@babel/types': 7.22.11 + babel-plugin-polyfill-corejs2: 0.4.5(@babel/core@7.22.11) + babel-plugin-polyfill-corejs3: 0.8.3(@babel/core@7.22.11) + babel-plugin-polyfill-regenerator: 0.5.2(@babel/core@7.22.11) + core-js-compat: 3.32.1 + semver: 6.3.1 transitivePeerDependencies: - supports-color dev: true - /@babel/preset-flow@7.18.6(@babel/core@7.21.3): - resolution: {integrity: sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==} + /@babel/preset-flow@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-ta2qZ+LSiGCrP5pgcGt8xMnnkXQrq8Sa4Ulhy06BOlF5QbLw9q5hIx7bn5MrsvyTGAfh6kTOo07Q+Pfld/8Y5Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-validator-option': 7.21.0 - '@babel/plugin-transform-flow-strip-types': 7.19.0(@babel/core@7.21.3) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.22.5 + '@babel/plugin-transform-flow-strip-types': 7.22.5(@babel/core@7.21.3) dev: true /@babel/preset-modules@0.1.5(@babel/core@7.21.3): @@ -2210,6 +3099,17 @@ packages: esutils: 2.0.3 dev: true + /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.22.11): + resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==} + peerDependencies: + '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/types': 7.22.11 + esutils: 2.0.3 + dev: true + /@babel/preset-react@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-zXr6atUmyYdiWRVLOZahakYmOBHtWc2WGCkP8PYTgZi0iJXDY2CN180TdrIW4OGOAdLc7TifzDIvtx6izaRIzg==} engines: {node: '>=6.9.0'} @@ -2239,8 +3139,8 @@ packages: - supports-color dev: true - /@babel/register@7.18.9(@babel/core@7.21.3): - resolution: {integrity: sha512-ZlbnXDcNYHMR25ITwwNKT88JiaukkdVj/nG7r3wnuXkOTHc60Uy05PwMCPre0hSkY68E6zK3xz+vUJSP2jWmcw==} + /@babel/register@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-vV6pm/4CijSQ8Y47RH5SopXzursN35RQINfGJkmOlcpAtGuf94miFvIPhCKGQN7WGIcsgG1BHEX2KVdTYwTwUQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 @@ -2249,10 +3149,14 @@ packages: clone-deep: 4.0.1 find-cache-dir: 2.1.0 make-dir: 2.1.0 - pirates: 4.0.5 + pirates: 4.0.6 source-map-support: 0.5.21 dev: true + /@babel/regjsgen@0.8.0: + resolution: {integrity: sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==} + dev: true + /@babel/runtime@7.20.7: resolution: {integrity: sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ==} engines: {node: '>=6.9.0'} @@ -2265,6 +3169,13 @@ packages: dependencies: regenerator-runtime: 0.14.0 + /@babel/runtime@7.22.11: + resolution: {integrity: sha512-ee7jVNlWN09+KftVOu9n7S8gQzD/Z6hN/I8VBRXW4P1+Xe7kJGXMwu8vds4aGIMHZnNbdpSWCfZZtinytpcAvA==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.0 + dev: true + /@babel/template@7.20.7: resolution: {integrity: sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==} engines: {node: '>=6.9.0'} @@ -2273,6 +3184,15 @@ packages: '@babel/parser': 7.21.3 '@babel/types': 7.21.3 + /@babel/template@7.22.5: + resolution: {integrity: sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.22.10 + '@babel/parser': 7.22.11 + '@babel/types': 7.22.11 + dev: true + /@babel/traverse@7.20.12(supports-color@5.5.0): resolution: {integrity: sha512-MsIbFN0u+raeja38qboyF8TIT7K0BFzz/Yd/77ta4MsUsmP2RAnidIlwq7d5HFQrH/OZJecGV6B71C4zAgpoSQ==} engines: {node: '>=6.9.0'} @@ -2308,17 +3228,35 @@ packages: transitivePeerDependencies: - supports-color - /@babel/types@7.20.7: - resolution: {integrity: sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==} + /@babel/traverse@7.22.11: + resolution: {integrity: sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-string-parser': 7.19.4 - '@babel/helper-validator-identifier': 7.19.1 - to-fast-properties: 2.0.0 - dev: true - - /@babel/types@7.21.3: - resolution: {integrity: sha512-sBGdETxC+/M4o/zKC0sl6sjWv62WFR/uzxrJ6uYyMLZOUlPnwzw0tKgVHOXxaAd5l2g8pEDM5RZ495GPQI77kg==} + '@babel/code-frame': 7.22.10 + '@babel/generator': 7.22.10 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.22.11 + '@babel/types': 7.22.11 + debug: 4.3.4(supports-color@5.5.0) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + dev: true + + /@babel/types@7.20.7: + resolution: {integrity: sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.19.4 + '@babel/helper-validator-identifier': 7.19.1 + to-fast-properties: 2.0.0 + dev: true + + /@babel/types@7.21.3: + resolution: {integrity: sha512-sBGdETxC+/M4o/zKC0sl6sjWv62WFR/uzxrJ6uYyMLZOUlPnwzw0tKgVHOXxaAd5l2g8pEDM5RZ495GPQI77kg==} engines: {node: '>=6.9.0'} dependencies: '@babel/helper-string-parser': 7.19.4 @@ -2343,6 +3281,15 @@ packages: to-fast-properties: 2.0.0 dev: true + /@babel/types@7.22.11: + resolution: {integrity: sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.22.5 + '@babel/helper-validator-identifier': 7.22.5 + to-fast-properties: 2.0.0 + dev: true + /@base2/pretty-print-object@1.0.1: resolution: {integrity: sha512-4iri8i1AqYHJE2DstZYkyEprg6Pq6sKx3xn5FpySk9sNhH7qN2LLlHJCfDTZRILNwQNPD7mATWM0TBui7uC1pA==} dev: true @@ -2598,6 +3545,15 @@ packages: react: '>=16.8.0' dependencies: react: 18.2.0 + dev: false + + /@emotion/use-insertion-effect-with-fallbacks@1.0.1(react@18.2.0): + resolution: {integrity: sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==} + peerDependencies: + react: '>=16.8.0' + dependencies: + react: 18.2.0 + dev: true /@emotion/utils@1.2.0: resolution: {integrity: sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw==} @@ -2616,6 +3572,15 @@ packages: dev: true optional: true + /@esbuild/android-arm64@0.18.20: + resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/android-arm@0.15.18: resolution: {integrity: sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==} engines: {node: '>=12'} @@ -2634,6 +3599,15 @@ packages: dev: true optional: true + /@esbuild/android-arm@0.18.20: + resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/android-x64@0.17.12: resolution: {integrity: sha512-m4OsaCr5gT+se25rFPHKQXARMyAehHTQAz4XX1Vk3d27VtqiX0ALMBPoXZsGaB6JYryCLfgGwUslMqTfqeLU0w==} engines: {node: '>=12'} @@ -2643,6 +3617,15 @@ packages: dev: true optional: true + /@esbuild/android-x64@0.18.20: + resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/darwin-arm64@0.17.12: resolution: {integrity: sha512-O3GCZghRIx+RAN0NDPhyyhRgwa19MoKlzGonIb5hgTj78krqp9XZbYCvFr9N1eUxg0ZQEpiiZ4QvsOQwBpP+lg==} engines: {node: '>=12'} @@ -2652,6 +3635,15 @@ packages: dev: true optional: true + /@esbuild/darwin-arm64@0.18.20: + resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /@esbuild/darwin-x64@0.17.12: resolution: {integrity: sha512-5D48jM3tW27h1qjaD9UNRuN+4v0zvksqZSPZqeSWggfMlsVdAhH3pwSfQIFJwcs9QJ9BRibPS4ViZgs3d2wsCA==} engines: {node: '>=12'} @@ -2661,6 +3653,15 @@ packages: dev: true optional: true + /@esbuild/darwin-x64@0.18.20: + resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /@esbuild/freebsd-arm64@0.17.12: resolution: {integrity: sha512-OWvHzmLNTdF1erSvrfoEBGlN94IE6vCEaGEkEH29uo/VoONqPnoDFfShi41Ew+yKimx4vrmmAJEGNoyyP+OgOQ==} engines: {node: '>=12'} @@ -2670,6 +3671,15 @@ packages: dev: true optional: true + /@esbuild/freebsd-arm64@0.18.20: + resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/freebsd-x64@0.17.12: resolution: {integrity: sha512-A0Xg5CZv8MU9xh4a+7NUpi5VHBKh1RaGJKqjxe4KG87X+mTjDE6ZvlJqpWoeJxgfXHT7IMP9tDFu7IZ03OtJAw==} engines: {node: '>=12'} @@ -2679,6 +3689,15 @@ packages: dev: true optional: true + /@esbuild/freebsd-x64@0.18.20: + resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-arm64@0.17.12: resolution: {integrity: sha512-cK3AjkEc+8v8YG02hYLQIQlOznW+v9N+OI9BAFuyqkfQFR+DnDLhEM5N8QRxAUz99cJTo1rLNXqRrvY15gbQUg==} engines: {node: '>=12'} @@ -2688,6 +3707,15 @@ packages: dev: true optional: true + /@esbuild/linux-arm64@0.18.20: + resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-arm@0.17.12: resolution: {integrity: sha512-WsHyJ7b7vzHdJ1fv67Yf++2dz3D726oO3QCu8iNYik4fb5YuuReOI9OtA+n7Mk0xyQivNTPbl181s+5oZ38gyA==} engines: {node: '>=12'} @@ -2697,6 +3725,15 @@ packages: dev: true optional: true + /@esbuild/linux-arm@0.18.20: + resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-ia32@0.17.12: resolution: {integrity: sha512-jdOBXJqcgHlah/nYHnj3Hrnl9l63RjtQ4vn9+bohjQPI2QafASB5MtHAoEv0JQHVb/xYQTFOeuHnNYE1zF7tYw==} engines: {node: '>=12'} @@ -2706,6 +3743,15 @@ packages: dev: true optional: true + /@esbuild/linux-ia32@0.18.20: + resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-loong64@0.15.18: resolution: {integrity: sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==} engines: {node: '>=12'} @@ -2724,6 +3770,15 @@ packages: dev: true optional: true + /@esbuild/linux-loong64@0.18.20: + resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-mips64el@0.17.12: resolution: {integrity: sha512-o8CIhfBwKcxmEENOH9RwmUejs5jFiNoDw7YgS0EJTF6kgPgcqLFjgoc5kDey5cMHRVCIWc6kK2ShUePOcc7RbA==} engines: {node: '>=12'} @@ -2733,6 +3788,15 @@ packages: dev: true optional: true + /@esbuild/linux-mips64el@0.18.20: + resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-ppc64@0.17.12: resolution: {integrity: sha512-biMLH6NR/GR4z+ap0oJYb877LdBpGac8KfZoEnDiBKd7MD/xt8eaw1SFfYRUeMVx519kVkAOL2GExdFmYnZx3A==} engines: {node: '>=12'} @@ -2742,6 +3806,15 @@ packages: dev: true optional: true + /@esbuild/linux-ppc64@0.18.20: + resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-riscv64@0.17.12: resolution: {integrity: sha512-jkphYUiO38wZGeWlfIBMB72auOllNA2sLfiZPGDtOBb1ELN8lmqBrlMiucgL8awBw1zBXN69PmZM6g4yTX84TA==} engines: {node: '>=12'} @@ -2751,6 +3824,15 @@ packages: dev: true optional: true + /@esbuild/linux-riscv64@0.18.20: + resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-s390x@0.17.12: resolution: {integrity: sha512-j3ucLdeY9HBcvODhCY4b+Ds3hWGO8t+SAidtmWu/ukfLLG/oYDMaA+dnugTVAg5fnUOGNbIYL9TOjhWgQB8W5g==} engines: {node: '>=12'} @@ -2760,6 +3842,15 @@ packages: dev: true optional: true + /@esbuild/linux-s390x@0.18.20: + resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-x64@0.17.12: resolution: {integrity: sha512-uo5JL3cgaEGotaqSaJdRfFNSCUJOIliKLnDGWaVCgIKkHxwhYMm95pfMbWZ9l7GeW9kDg0tSxcy9NYdEtjwwmA==} engines: {node: '>=12'} @@ -2769,6 +3860,15 @@ packages: dev: true optional: true + /@esbuild/linux-x64@0.18.20: + resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/netbsd-x64@0.17.12: resolution: {integrity: sha512-DNdoRg8JX+gGsbqt2gPgkgb00mqOgOO27KnrWZtdABl6yWTST30aibGJ6geBq3WM2TIeW6COs5AScnC7GwtGPg==} engines: {node: '>=12'} @@ -2778,6 +3878,15 @@ packages: dev: true optional: true + /@esbuild/netbsd-x64@0.18.20: + resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/openbsd-x64@0.17.12: resolution: {integrity: sha512-aVsENlr7B64w8I1lhHShND5o8cW6sB9n9MUtLumFlPhG3elhNWtE7M1TFpj3m7lT3sKQUMkGFjTQBrvDDO1YWA==} engines: {node: '>=12'} @@ -2787,6 +3896,15 @@ packages: dev: true optional: true + /@esbuild/openbsd-x64@0.18.20: + resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/sunos-x64@0.17.12: resolution: {integrity: sha512-qbHGVQdKSwi0JQJuZznS4SyY27tYXYF0mrgthbxXrZI3AHKuRvU+Eqbg/F0rmLDpW/jkIZBlCO1XfHUBMNJ1pg==} engines: {node: '>=12'} @@ -2796,6 +3914,15 @@ packages: dev: true optional: true + /@esbuild/sunos-x64@0.18.20: + resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-arm64@0.17.12: resolution: {integrity: sha512-zsCp8Ql+96xXTVTmm6ffvoTSZSV2B/LzzkUXAY33F/76EajNw1m+jZ9zPfNJlJ3Rh4EzOszNDHsmG/fZOhtqDg==} engines: {node: '>=12'} @@ -2805,6 +3932,15 @@ packages: dev: true optional: true + /@esbuild/win32-arm64@0.18.20: + resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-ia32@0.17.12: resolution: {integrity: sha512-FfrFjR4id7wcFYOdqbDfDET3tjxCozUgbqdkOABsSFzoZGFC92UK7mg4JKRc/B3NNEf1s2WHxJ7VfTdVDPN3ng==} engines: {node: '>=12'} @@ -2814,6 +3950,15 @@ packages: dev: true optional: true + /@esbuild/win32-ia32@0.18.20: + resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-x64@0.17.12: resolution: {integrity: sha512-JOOxw49BVZx2/5tW3FqkdjSD/5gXYeVGPDcB0lvap0gLQshkh1Nyel1QazC+wNxus3xPlsYAgqU1BUmrmCvWtw==} engines: {node: '>=12'} @@ -2823,6 +3968,15 @@ packages: dev: true optional: true + /@esbuild/win32-x64@0.18.20: + resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@eslint-community/eslint-utils@4.3.0(eslint@8.36.0): resolution: {integrity: sha512-v3oplH6FYCULtFuCeqyuTd9D2WKO937Dxdq+GmHOLL72TTRriLxz2VLlNfkZRsvj6PKnOPAtuT6dwrs/pA5DvA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3337,6 +4491,12 @@ packages: resolution: {integrity: sha512-EvYTiXet5XqweYGClEmpu3BoxmsQ4hkj3QaYA6qEnigCWffTP3vNRwBReTdrwDwo7OoJ3wM8Uoe9Uk4n+d4hfg==} dev: false + /@floating-ui/core@1.4.1: + resolution: {integrity: sha512-jk3WqquEJRlcyu7997NtR5PibI+y5bi+LS3hPmguVClypenMsCY3CBa3LAQnozRCtCrYWSEtAdiskpamuJRFOQ==} + dependencies: + '@floating-ui/utils': 0.1.1 + dev: true + /@floating-ui/dom@1.1.0: resolution: {integrity: sha512-TSogMPVxbRe77QCj1dt8NmRiJasPvuc+eT5jnJ6YpLqgOD2zXc5UA3S1qwybN+GVCDNdKfpKy1oj8RpzLJvh6A==} dependencies: @@ -3349,6 +4509,13 @@ packages: '@floating-ui/core': 1.2.6 dev: false + /@floating-ui/dom@1.5.1: + resolution: {integrity: sha512-KwvVcPSXg6mQygvA1TjbN/gh///36kKtllIF8SUm0qpFj8+rvYrpvlYdL1JoA71SHpDqgSSdGOSoQ0Mp3uY5aw==} + dependencies: + '@floating-ui/core': 1.4.1 + '@floating-ui/utils': 0.1.1 + dev: true + /@floating-ui/react-dom@1.2.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-YCLlqibZtgUhxUpxkSp1oekvYgH/jI4KdZEJv85E62twlZHN43xdlQNe6JcF4ROD3/Zu6juNHN+aOygN+6yZjg==} peerDependencies: @@ -3371,6 +4538,17 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false + /@floating-ui/react-dom@2.0.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-5qhlDvjaLmAst/rKb3VdlCinwTF4EYMiVxuuc/HVUjs46W0zgtbMmAZ1UTsDrRTxRmUEzl92mOtWbeeXL26lSQ==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + dependencies: + '@floating-ui/dom': 1.5.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + /@floating-ui/react@0.19.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==} peerDependencies: @@ -3384,6 +4562,10 @@ packages: tabbable: 6.1.2 dev: false + /@floating-ui/utils@0.1.1: + resolution: {integrity: sha512-m0G6wlnhm/AX0H12IOWtK8gASEMffnX08RtKkCgTdHb9JpHKGloI7icFfLg9ZmQeavcvR0PKmzxClyuFPSjKWw==} + dev: true + /@formatjs/ecma402-abstract@1.14.3: resolution: {integrity: sha512-SlsbRC/RX+/zg4AApWIFNDdkLtFbkq3LNoZWXZCE/nHVKqoIJyaoQyge/I0Y38vLxowUn9KTtXgusLD91+orbg==} dependencies: @@ -3638,6 +4820,18 @@ packages: - encoding dev: true + /@isaacs/cliui@8.0.2: + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + dependencies: + string-width: 5.1.2 + string-width-cjs: /string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: /strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: /wrap-ansi@7.0.0 + dev: true + /@istanbuljs/load-nyc-config@1.1.0: resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -3833,6 +5027,13 @@ packages: '@sinclair/typebox': 0.27.8 dev: true + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: true + /@jest/source-map@29.6.0: resolution: {integrity: sha512-oA+I2SHHQGxDCZpbrsCQSoMLb3Bz547JnM+jUr9qEbuw0vQlWZfpPS7CO9J7XiwKicEz9OFn/IYoLkkiUD7bzA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3908,6 +5109,29 @@ packages: - supports-color dev: true + /@jest/transform@29.6.4: + resolution: {integrity: sha512-8thgRSiXUqtr/pPGY/OsyHuMjGyhVnWrFAwoxmIemlBuiMyU1WFs0tXoNxzcr4A4uErs/ABre76SGmrr5ab/AA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@babel/core': 7.21.3 + '@jest/types': 29.6.3 + '@jridgewell/trace-mapping': 0.3.19 + babel-plugin-istanbul: 6.1.1 + chalk: 4.1.2 + convert-source-map: 2.0.0 + fast-json-stable-stringify: 2.1.0 + graceful-fs: 4.2.11 + jest-haste-map: 29.6.4 + jest-regex-util: 29.6.3 + jest-util: 29.6.3 + micromatch: 4.0.5 + pirates: 4.0.6 + slash: 3.0.0 + write-file-atomic: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: true + /@jest/types@29.5.0: resolution: {integrity: sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3932,6 +5156,18 @@ packages: chalk: 4.1.2 dev: true + /@jest/types@29.6.3: + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.4 + '@types/istanbul-reports': 3.0.1 + '@types/node': 18.16.3 + '@types/yargs': 17.0.24 + chalk: 4.1.2 + dev: true + /@joshwooding/vite-plugin-react-docgen-typescript@0.2.1(typescript@5.0.2)(vite@4.2.0): resolution: {integrity: sha512-ou4ZJSXMMWHqGS4g8uNRbC5TiTWxAgQZiVucoUrOCWuPrTbkpJbmVyIi9jU72SBry7gQtuMEDp4YR8EEXAg7VQ==} peerDependencies: @@ -4047,12 +5283,12 @@ packages: '@types/whatwg-streams': 0.0.7 dev: false - /@mdx-js/react@2.2.1(react@18.2.0): - resolution: {integrity: sha512-YdXcMcEnqZhzql98RNrqYo9cEhTTesBiCclEtoiQUbJwx87q9453GTapYU6kJ8ZZ2ek1Vp25SiAXEFy5O/eAPw==} + /@mdx-js/react@2.3.0(react@18.2.0): + resolution: {integrity: sha512-zQH//gdOmuu7nt2oJR29vFhDv88oGPmVw6BggmrHeMI+xgEkp1B2dX9/bMBSYtK0dyLX/aOmesKS09g222K1/g==} peerDependencies: react: '>=16' dependencies: - '@types/mdx': 2.0.3 + '@types/mdx': 2.0.7 '@types/react': 18.2.20 react: 18.2.0 dev: true @@ -4140,8 +5376,8 @@ packages: tslib: 2.5.2 dev: false - /@ndelangen/get-tarball@3.0.7: - resolution: {integrity: sha512-NqGfTZIZpRFef1GoVaShSSRwDC3vde3ThtTeqFdcYd6ipKqnfEVhjK2hUeHjCQUcptyZr2TONqcloFXM+5QBrQ==} + /@ndelangen/get-tarball@3.0.9: + resolution: {integrity: sha512-9JKTEik4vq+yGosHYhZ1tiH/3WpUS0Nh0kej4Agndhox8pAdWhEx5knFVRcb/ya9knCRCs1rPxNrSXTDdfVqpA==} dependencies: gunzip-maybe: 1.4.2 pump: 3.0.0 @@ -4162,172 +5398,739 @@ packages: run-parallel: 1.2.0 dev: true - /@nodelib/fs.stat@2.0.5: - resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} - engines: {node: '>= 8'} + /@nodelib/fs.stat@2.0.5: + resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} + engines: {node: '>= 8'} + dev: true + + /@nodelib/fs.walk@1.2.8: + resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} + engines: {node: '>= 8'} + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.15.0 + dev: true + + /@orval/angular@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-KdE9Bp3eRivrumGSlukrlIGTxD1iLcdAnxogqeVtTS27z1c4SOOpdjLzWT0VlyGLgVdYe+HiHLLhPR48CI8dXg==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/axios@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-PGKCiyGwv/CiywPdP78zb/Hm5qsK37tlh8MwCd+38uU6B1VsZ2lfDLMcAsXe9fgrEqRmpEkgvyr6JTcilE904A==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/core@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-Rh6GVpWnCKdCMAe6vdlFMhrTkZo4EIkNkWS2b5T+4PHRsQoTA98jhlGXLsD38hG2tW9+YJfibNBG4xnIYFeFhA==} + dependencies: + '@apidevtools/swagger-parser': 10.1.0(openapi-types@12.1.3) + acorn: 8.8.2 + ajv: 8.12.0 + chalk: 4.1.2 + compare-versions: 4.1.4 + debug: 4.3.4(supports-color@5.5.0) + esbuild: 0.15.18 + esutils: 2.0.3 + fs-extra: 10.1.0 + globby: 11.1.0 + ibm-openapi-validator: 0.97.5 + lodash.get: 4.4.2 + lodash.isempty: 4.4.0 + lodash.omit: 4.5.0 + lodash.uniq: 4.5.0 + lodash.uniqby: 4.7.0 + lodash.uniqwith: 4.5.0 + micromatch: 4.0.5 + openapi3-ts: 3.2.0 + swagger2openapi: 7.0.8 + validator: 13.9.0 + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/msw@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-iKfpp6ixxTBraCnNHeqi1z+Y5Gkr1USM2vLWD3LDWS3kLUqgWbZ2SiL3jma1ja9sWUYpRCFU3TepiYSSMoFYxg==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + cuid: 2.1.8 + lodash.get: 4.4.2 + lodash.omit: 4.5.0 + openapi3-ts: 3.2.0 + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/query@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-+aHTJSj8qzm00tU0F5++5NVbFBtcWeKV2MdLD6jCSFGQ4EssbOaoXE8noaw4rlX66P76g3vN8awTsf8BOz5hHQ==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + lodash.omitby: 4.6.0 + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/swr@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-Bx1a7RA+6eYejdnnPRyLlaXu8ixlYtkhx7P+Qtk3VBNcDH8O0KbVmrpCNpDYHVuIIPSpQHJjX3GtXWKwP7gmCA==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@orval/zod@6.15.0(openapi-types@12.1.3): + resolution: {integrity: sha512-8BfOOfOyxX3Fei0BnrW72CCTY7TWeUXQCvL7eWakiD5sMNmm1FcaCQ6JZwBPX3sGlWR1ElYmRJisd5ihZlaCsg==} + dependencies: + '@orval/core': 6.15.0(openapi-types@12.1.3) + lodash.uniq: 4.5.0 + transitivePeerDependencies: + - encoding + - openapi-types + - supports-color + dev: true + + /@pkgjs/parseargs@0.11.0: + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + requiresBuild: true + dev: true + optional: true + + /@pkgr/utils@2.3.1: + resolution: {integrity: sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==} + engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + dependencies: + cross-spawn: 7.0.3 + is-glob: 4.0.3 + open: 8.4.0 + picocolors: 1.0.0 + tiny-glob: 0.2.9 + tslib: 2.5.2 + dev: true + + /@popperjs/core@2.11.6: + resolution: {integrity: sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==} + + /@protobufjs/aspromise@1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + dev: false + + /@protobufjs/base64@1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + dev: false + + /@protobufjs/codegen@2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + dev: false + + /@protobufjs/eventemitter@1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + dev: false + + /@protobufjs/fetch@1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + dev: false + + /@protobufjs/float@1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + dev: false + + /@protobufjs/inquire@1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + dev: false + + /@protobufjs/path@1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + dev: false + + /@protobufjs/pool@1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + dev: false + + /@protobufjs/utf8@1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + dev: false + + /@radix-ui/number@1.0.1: + resolution: {integrity: sha512-T5gIdVO2mmPW3NNhjNgEP3cqMXjXL9UbO0BzWcXfvdBs+BohbQxvd/K5hSVKmn9/lbTdsQVKbUcP5WLCwvUbBg==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true + + /@radix-ui/primitive@1.0.1: + resolution: {integrity: sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true + + /@radix-ui/react-arrow@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-collection@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-3SzW+0PW7yBBoQlT8wNcGtaxaD0XSu0uLUFgrtHY08Acx05TaHaOmVLR73c0j/cqpDy53KBMO7s0dx2wmOIDIA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-compose-refs@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-context@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-direction@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-RXcvnXgyvYvBEOhCBuddKecVkoMiI10Jcm5cTI7abJRAHYfFxeu+FBQs/DvdxSYucxR5mna0dNsL6QFlds5TMA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-dismissable-layer@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-7UpBa/RKMoHJYjie1gkF1DlK8l1fdU/VKDpoS3rCCo8YBJR294GwcEHyxHw72yvphJ7ld0AXEcSLAzY2F/WyCg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-focus-guards@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-focus-scope@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-upXdPfqI4islj2CslyfUBNlaJCPybbqRHAi1KER7Isel9Q2AtSJ0zRBZv8mWQiFXD2nyAJ4BhC3yXgZ6kMBSrQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-id@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-popper@1.1.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-1CnGGfFi/bbqtJZZ0P/NQY20xdG3E0LALJaLUEoKwPLwl6PPPfbeiCqMVQnhoFRAxjJj4RpBRJzDmUgsex2tSg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@floating-ui/react-dom': 2.0.2(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-rect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-size': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/rect': 1.0.1 + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-portal@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-xLYZeHrWoPmA5mEKEfZZevoVRK/Q43GfzRXkWV6qawIWWK8t6ifIiLQdd7rmQ4Vk1bmI21XhqF9BN3jWf+phpA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-primitive@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-roving-focus@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-2mUg5Mgcu001VkGy+FfzZyzbmuUWzgWkj3rvv4yu+mLw03+mTzbxZHvfcGyFp2b8EkQeMkpRQ5FiA2Vr2O6TeQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-id': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-select@1.2.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-zI7McXr8fNaSrUY9mZe4x/HC0jTLY9fWNhO1oLWYMQGDXuV4UCivIGTxwioSzO0ZCYX9iSLyWmAh/1TOmX3Cnw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/number': 1.0.1 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-dismissable-layer': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-focus-guards': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-focus-scope': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-id': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-popper': 1.1.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-portal': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-previous': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-visually-hidden': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + aria-hidden: 1.2.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-remove-scroll: 2.5.5(@types/react@18.2.20)(react@18.2.0) + dev: true + + /@radix-ui/react-separator@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-itYmTy/kokS21aiV5+Z56MZB54KrhPgn6eHDKkFeOLR34HMN2s8PaN47qZZAGnvupcjxHaFZnW4pQEh0BvvVuw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-slot@1.0.2(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-toggle-group@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toggle': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-toggle@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) dev: true - /@nodelib/fs.walk@1.2.8: - resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} - engines: {node: '>= 8'} + /@radix-ui/react-toolbar@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-tBgmM/O7a07xbaEkYJWYTXkIdU/1pW4/KZORR43toC/4XWyBCURK0ei9kMUdp+gTPPKBgYLxXmRSH1EVcIDp8Q==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true dependencies: - '@nodelib/fs.scandir': 2.1.5 - fastq: 1.15.0 + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-separator': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toggle-group': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) dev: true - /@orval/angular@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-KdE9Bp3eRivrumGSlukrlIGTxD1iLcdAnxogqeVtTS27z1c4SOOpdjLzWT0VlyGLgVdYe+HiHLLhPR48CI8dXg==} + /@radix-ui/react-use-callback-ref@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/axios@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-PGKCiyGwv/CiywPdP78zb/Hm5qsK37tlh8MwCd+38uU6B1VsZ2lfDLMcAsXe9fgrEqRmpEkgvyr6JTcilE904A==} + /@radix-ui/react-use-controllable-state@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/core@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-Rh6GVpWnCKdCMAe6vdlFMhrTkZo4EIkNkWS2b5T+4PHRsQoTA98jhlGXLsD38hG2tW9+YJfibNBG4xnIYFeFhA==} + /@radix-ui/react-use-escape-keydown@1.0.3(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@apidevtools/swagger-parser': 10.1.0(openapi-types@12.1.3) - acorn: 8.8.2 - ajv: 8.12.0 - chalk: 4.1.2 - compare-versions: 4.1.4 - debug: 4.3.4(supports-color@5.5.0) - esbuild: 0.15.18 - esutils: 2.0.3 - fs-extra: 10.1.0 - globby: 11.1.0 - ibm-openapi-validator: 0.97.5 - lodash.get: 4.4.2 - lodash.isempty: 4.4.0 - lodash.omit: 4.5.0 - lodash.uniq: 4.5.0 - lodash.uniqby: 4.7.0 - lodash.uniqwith: 4.5.0 - micromatch: 4.0.5 - openapi3-ts: 3.2.0 - swagger2openapi: 7.0.8 - validator: 13.9.0 - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/msw@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-iKfpp6ixxTBraCnNHeqi1z+Y5Gkr1USM2vLWD3LDWS3kLUqgWbZ2SiL3jma1ja9sWUYpRCFU3TepiYSSMoFYxg==} + /@radix-ui/react-use-layout-effect@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - cuid: 2.1.8 - lodash.get: 4.4.2 - lodash.omit: 4.5.0 - openapi3-ts: 3.2.0 - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/query@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-+aHTJSj8qzm00tU0F5++5NVbFBtcWeKV2MdLD6jCSFGQ4EssbOaoXE8noaw4rlX66P76g3vN8awTsf8BOz5hHQ==} + /@radix-ui/react-use-previous@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-cV5La9DPwiQ7S0gf/0qiD6YgNqM5Fk97Kdrlc5yBcrF3jyEZQwm7vYFqMo4IfeHgJXsRaMvLABFtd0OVEmZhDw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - lodash.omitby: 4.6.0 - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/swr@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-Bx1a7RA+6eYejdnnPRyLlaXu8ixlYtkhx7P+Qtk3VBNcDH8O0KbVmrpCNpDYHVuIIPSpQHJjX3GtXWKwP7gmCA==} + /@radix-ui/react-use-rect@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@radix-ui/rect': 1.0.1 + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@orval/zod@6.15.0(openapi-types@12.1.3): - resolution: {integrity: sha512-8BfOOfOyxX3Fei0BnrW72CCTY7TWeUXQCvL7eWakiD5sMNmm1FcaCQ6JZwBPX3sGlWR1ElYmRJisd5ihZlaCsg==} + /@radix-ui/react-use-size@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true dependencies: - '@orval/core': 6.15.0(openapi-types@12.1.3) - lodash.uniq: 4.5.0 - transitivePeerDependencies: - - encoding - - openapi-types - - supports-color + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 dev: true - /@pkgr/utils@2.3.1: - resolution: {integrity: sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==} - engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + /@radix-ui/react-visually-hidden@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true dependencies: - cross-spawn: 7.0.3 - is-glob: 4.0.3 - open: 8.4.0 - picocolors: 1.0.0 - tiny-glob: 0.2.9 - tslib: 2.5.2 + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) dev: true - /@popperjs/core@2.11.6: - resolution: {integrity: sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==} - - /@protobufjs/aspromise@1.1.2: - resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} - dev: false - - /@protobufjs/base64@1.1.2: - resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} - dev: false - - /@protobufjs/codegen@2.0.4: - resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} - dev: false - - /@protobufjs/eventemitter@1.1.0: - resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} - dev: false - - /@protobufjs/fetch@1.1.0: - resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + /@radix-ui/rect@1.0.1: + resolution: {integrity: sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==} dependencies: - '@protobufjs/aspromise': 1.1.2 - '@protobufjs/inquire': 1.1.0 - dev: false - - /@protobufjs/float@1.0.2: - resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} - dev: false - - /@protobufjs/inquire@1.1.0: - resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} - dev: false - - /@protobufjs/path@1.1.2: - resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} - dev: false - - /@protobufjs/pool@1.1.0: - resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} - dev: false - - /@protobufjs/utf8@1.1.0: - resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} - dev: false + '@babel/runtime': 7.22.11 + dev: true /@remix-run/router@1.7.2: resolution: {integrity: sha512-7Lcn7IqGMV+vizMPoEl5F0XDshcdDYtMI6uJLQdQz5CfZAwy3vvGKYSUk789qndt5dEC4HfSjviSYlSoHGL2+A==} @@ -4362,16 +6165,23 @@ packages: rollup: 2.79.1 dev: true - /@rollup/pluginutils@4.2.1: - resolution: {integrity: sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==} - engines: {node: '>= 8.0.0'} + /@rollup/pluginutils@5.0.2(rollup@2.79.1): + resolution: {integrity: sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0 + peerDependenciesMeta: + rollup: + optional: true dependencies: + '@types/estree': 1.0.0 estree-walker: 2.0.2 picomatch: 2.3.1 + rollup: 2.79.1 dev: true - /@rollup/pluginutils@5.0.2(rollup@2.79.1): - resolution: {integrity: sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==} + /@rollup/pluginutils@5.0.4(rollup@2.79.1): + resolution: {integrity: sha512-0KJnIoRI8A+a1dqOYLxH8vBf8bphDmty5QvIm2hqm7oFCFYKCAZWWd2hXgMibaPsNDhI0AtpYfQZJG47pt/k4g==} engines: {node: '>=14.0.0'} peerDependencies: rollup: ^1.20.0||^2.0.0||^3.0.0 @@ -4379,7 +6189,7 @@ packages: rollup: optional: true dependencies: - '@types/estree': 1.0.0 + '@types/estree': 1.0.1 estree-walker: 2.0.2 picomatch: 2.3.1 rollup: 2.79.1 @@ -4468,7 +6278,7 @@ packages: fast-memoize: 2.5.2 immer: 9.0.21 lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 urijs: 1.19.11 dev: true @@ -4519,7 +6329,7 @@ packages: stacktracey: 2.1.8 strip-ansi: 6.0.1 text-table: 0.2.0 - tslib: 2.5.2 + tslib: 2.6.2 yargs: 17.3.1 transitivePeerDependencies: - encoding @@ -4550,7 +6360,7 @@ packages: nimma: 0.2.2 pony-cause: 1.1.1 simple-eval: 1.0.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4562,7 +6372,7 @@ packages: '@stoplight/json': 3.20.2 '@stoplight/spectral-core': 1.18.0 '@types/json-schema': 7.0.11 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4581,7 +6391,7 @@ packages: ajv-errors: 3.0.0(ajv@8.12.0) ajv-formats: 2.1.1(ajv@8.12.0) lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4593,7 +6403,7 @@ packages: '@stoplight/json': 3.20.2 '@stoplight/types': 13.15.0 '@stoplight/yaml': 4.2.3 - tslib: 2.5.2 + tslib: 2.6.2 dev: true /@stoplight/spectral-ref-resolver@1.0.2: @@ -4604,7 +6414,7 @@ packages: '@stoplight/json-ref-resolver': 3.1.5 '@stoplight/spectral-runtime': 1.1.2 dependency-graph: 0.11.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4617,7 +6427,7 @@ packages: '@stoplight/json-ref-resolver': 3.1.5 '@stoplight/spectral-runtime': 1.1.2 dependency-graph: 0.11.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4640,7 +6450,7 @@ packages: '@types/node': 18.16.3 pony-cause: 1.1.1 rollup: 2.79.1 - tslib: 2.5.2 + tslib: 2.6.2 validate-npm-package-name: 3.0.0 transitivePeerDependencies: - encoding @@ -4662,7 +6472,7 @@ packages: ast-types: 0.14.2 astring: 1.8.4 reserved: 0.1.2 - tslib: 2.5.2 + tslib: 2.6.2 validate-npm-package-name: 3.0.0 transitivePeerDependencies: - encoding @@ -4685,7 +6495,7 @@ packages: ajv-formats: 2.1.1(ajv@8.12.0) json-schema-traverse: 1.0.0 lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4700,7 +6510,7 @@ packages: abort-controller: 3.0.0 lodash: 4.17.21 node-fetch: 2.6.9 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4740,11 +6550,11 @@ packages: '@stoplight/ordered-object-literal': 1.0.4 '@stoplight/types': 13.15.0 '@stoplight/yaml-ast-parser': 0.0.48 - tslib: 2.5.2 + tslib: 2.6.2 dev: true - /@storybook/addon-actions@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-rcj39u9MrmzsrDWYt1zsoVxrogZ1Amrv9xkEofEY/QKUr2R3xpHhTALveY9BKIlG1GoE8zLlLoP2k4nz3sNNwQ==} + /@storybook/addon-actions@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-TsTOHGmwBHRsWS9kaG/bu6haP2dMeiETeGwOgfB5qmukodenXlmi1RujtUdJCNwW3APa0utEFYFKtZVEu9f7WQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4754,28 +6564,31 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 dequal: 2.0.3 lodash: 4.17.21 polished: 4.2.2 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - react-inspector: 6.0.1(react@18.2.0) - telejson: 7.0.4 + react-inspector: 6.0.2(react@18.2.0) + telejson: 7.2.0 ts-dedent: 2.2.0 - uuid-browser: 3.1.0 + uuid: 9.0.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-backgrounds@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-yRNHQ4PPRJ+HIORQPhDGxn5xolw1xW0ByQZoNRpMD+AMEyfUNFdWbCsRQAOWjNhawxVMHM7EeA2Exrb41zhEjA==} + /@storybook/addon-backgrounds@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-tcQSt6mjAR1h1XiMFlg9OvpAwvBCjFrtpr9qnVaOZD15EIu/TRoumkJOVA7J5sWuQ6kGJXx1t8FfhQfAqvJ9iw==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4785,22 +6598,25 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 memoizerific: 1.11.3 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-controls@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-dMpRtj5cmfC9vEMve5ncvbWCEC+WD9YuzJ+grdc48E/Hd//p+O2FE6klSkrz5FAjrc+rHINixdyssekpEL6nYQ==} + /@storybook/addon-controls@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-n9ZoxlV8c9VLNfpFY1HpcRxjUFmHPmcFnW0UzFfGknIArPKFxzw9S/zCJ7CSH9Mf7+NJtYAUzCXlSU/YzT1eZQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4810,97 +6626,100 @@ packages: react-dom: optional: true dependencies: - '@storybook/blocks': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-common': 7.0.2 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/blocks': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-common': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 lodash: 4.17.21 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-docs@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-q3rDWoZEym6Lkmhqc/HBNfLDAmTY8l0WINGUZo/nF98eP5iu4B7Nk7V6BRGYGQt6Y6ZyIQ8WKH0e/eJww2zIog==} + /@storybook/addon-docs@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-g4B+gM7xzRvUeiUcijPyxwDG/LlgHrfQx1chzY7oiFIImGXyewZ+CtGCjhrSdJGhXSj/69oqoz26RQ1VhSlrXg==} peerDependencies: - '@storybook/mdx1-csf': '>=1.0.0-0' react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 - peerDependenciesMeta: - '@storybook/mdx1-csf': - optional: true dependencies: - '@babel/core': 7.21.3 - '@babel/plugin-transform-react-jsx': 7.20.7(@babel/core@7.21.3) - '@jest/transform': 29.5.0 - '@mdx-js/react': 2.2.1(react@18.2.0) - '@storybook/blocks': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/csf-plugin': 7.0.2 - '@storybook/csf-tools': 7.0.2 + '@jest/transform': 29.6.4 + '@mdx-js/react': 2.3.0(react@18.2.0) + '@storybook/blocks': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/csf-plugin': 7.3.2 + '@storybook/csf-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/mdx2-csf': 1.0.0 - '@storybook/node-logger': 7.0.2 - '@storybook/postinstall': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/react-dom-shim': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 - fs-extra: 11.1.0 + '@storybook/mdx2-csf': 1.1.0 + '@storybook/node-logger': 7.3.2 + '@storybook/postinstall': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/react-dom-shim': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 + fs-extra: 11.1.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) remark-external-links: 8.0.0 remark-slug: 6.1.0 ts-dedent: 2.2.0 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-essentials@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-LAsWsXa/Pp2B4Ve2WVgc990FtsiHpFDRsq7S3V7xRrZP8DYRbtJIVdszPMDS5uKC+yzbswFEXz08lqbGvq8zgQ==} + /@storybook/addon-essentials@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-MI5wi5k/nDgAqnsS4/uibcQhMk3/mVkAAWNO+Epmg5UMCCmDch8SoX9BprEHARwwsVwXChiHAx99fXF/XacWFQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/addon-actions': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-backgrounds': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-controls': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-docs': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-highlight': 7.0.2 - '@storybook/addon-measure': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-outline': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-toolbars': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-viewport': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-common': 7.0.2 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 + '@storybook/addon-actions': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-backgrounds': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-controls': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-docs': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-highlight': 7.3.2 + '@storybook/addon-measure': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-outline': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-toolbars': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-viewport': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-common': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 transitivePeerDependencies: - - '@storybook/mdx1-csf' + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-highlight@7.0.2: - resolution: {integrity: sha512-9BkL1OOanguuy73S6nLK0isUb045tOkFONd/PQldOJ0PV3agCvKxKHyzlBz7Hsba8KZhY5jQs+nVW2NiREyGYg==} + /@storybook/addon-highlight@7.3.2: + resolution: {integrity: sha512-Zdq//ZqOYpm+xXHt00l0j/baVuZDSkpP6Xbd3jqXV1ToojAjANlk0CAzHCJxZBiyeSCj7Qxtj9LvTqD+IU/bMA==} dependencies: - '@storybook/core-events': 7.0.2 + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/preview-api': 7.0.2 + '@storybook/preview-api': 7.3.2 dev: true - /@storybook/addon-links@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-lPtfy2MqrcI9YjupBM2eRKGPdFKVPCz7WgO/JQQakGugORJTEGCyJrNJNtWY9jDenv8ynLZ40OxtPBZi54Sr6Q==} + /@storybook/addon-links@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-xpOpb33KscvmM2Sl9nFqU3DCk3tGaoqtFKkDOzf/QlZsMq9CCn4zPNGMfOFqifBEnDGDADHbp+Uxst5i535vdQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4910,22 +6729,22 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/router': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/router': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 dev: true - /@storybook/addon-measure@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-cf/d5MXpHAjyUiDIVfc8pLn79CPHgnryDmNNlSiP2zEFKcivrRWiu8Rmrad8pGqLkuAh+PXLKCGn9uiqDvg7QQ==} + /@storybook/addon-measure@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-bEoH3zuKA9b5RA0LBQzdSnoaxEKHa5rZDoAuMbKiEYotTqO7PfP2j/hil31F95UgmH7wPnSkRSqsBsUtWJz3Jg==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4935,19 +6754,23 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + tiny-invariant: 1.3.1 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-outline@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-thVISO4NM22xlETisBvAPvz2yFD3qLGOjgzBmj8l8r9Rv0IEdwdPrwm5j0WTv8OtbhC4A8lPpvMsn5FhY5mDXg==} + /@storybook/addon-outline@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-DA/O5b4bznV2JsC/o0/JkP2tZLLPftRaz2HHCG+z0mwzNv2pl8lvIl4RpIVJWt1iO0K17kT43ToYYjknMUdJnA==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4957,20 +6780,23 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-toolbars@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-tAxZ2+nUYsJdT1sx3BrmoMAZFM19+OzWJY6qSnbEq5zoRgvGZaXGR6tLMKydDoHQBU9Ta9YHGo7N7u7h1C23yg==} + /@storybook/addon-toolbars@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-hd+5Ax7p3vmsNNuO3t4pcmB2pxp58i9k12ygD66NLChSNafHxediLqdYJDTRuono2No1InV1HMZghlXXucCCHQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4980,17 +6806,20 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-viewport@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-TaHJWIIazPM/TerRbka9RqjMPNpwaRsGRdVRBtVoVosy1FzsEjAdQSO7RBMe4G03m5CacSqdsDiJCblI2AXaew==} + /@storybook/addon-viewport@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-G7i67xL35WE6qSmEoctavZUoPd2VDTaAqkRwrGa4oDQs5wed76PgIL2S5IybzbypSzPIXauiNQiBBd2RRMrLFg==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5000,182 +6829,210 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) memoizerific: 1.11.3 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/blocks@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-JzHmU8jZLzeQ6bunzci8j/2Ji18GBTyhrPFLk5RjEbMNGWpGjvER/yR127tZOdbPguVNr4iVbRfGzd1wGHlrzA==} + /@storybook/addons@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-qYwHniTJzfR7jKh5juYCjU9ukG7l1YAAt7BpnouItgRutxU/+UoC2iAFooQW+i74SxDoovqnEp9TkG7TAFOLxQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 - '@storybook/docs-tools': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/api@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-HAiaEl9uFQJM3AC5LhdnUbqr+7BVMaCNzhbUg1sWfO7sTFXPO0P1BAz9UuDKPlndwaVGcGpypRw9P/bdpuWLfA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + dependencies: + '@storybook/client-logger': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/blocks@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-j/PRnvGLn0Y3VAu/t6RrU7pjenb7II7Cl/SnFW8LzjMBKXBrkFaq8BRbglzDAUtGdAa9HmJBosogenoZ9iWoBw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/docs-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 '@types/lodash': 4.14.191 color-convert: 2.0.1 dequal: 2.0.3 lodash: 4.17.21 - markdown-to-jsx: 7.1.8(react@18.2.0) + markdown-to-jsx: 7.3.2(react@18.2.0) memoizerific: 1.11.3 polished: 4.2.2 react: 18.2.0 react-colorful: 5.6.1(react-dom@18.2.0)(react@18.2.0) react-dom: 18.2.0(react@18.2.0) - telejson: 7.0.4 + telejson: 7.2.0 + tocbot: 4.21.1 ts-dedent: 2.2.0 util-deprecate: 1.0.2 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/builder-manager@7.0.2: - resolution: {integrity: sha512-Oej/n8D7eaWgmWF7nN2hXLRM53lcYOdh6umSN8Mh/LcYUfxB+dvUBFzUjoLE0xjhW6xRinrKrENT5LcP/f/HBQ==} + /@storybook/builder-manager@7.3.2: + resolution: {integrity: sha512-M0zdzpnZSg6Gd/QiIbOJkVoifAADpMT85NOC5zuAg3h3o29hedVBAigv/CE2nSbuwZtqPifjxs1AUh7wgtmj8A==} dependencies: '@fal-works/esbuild-plugin-global-externals': 2.1.2 - '@storybook/core-common': 7.0.2 - '@storybook/manager': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@types/ejs': 3.1.1 + '@storybook/core-common': 7.3.2 + '@storybook/manager': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@types/ejs': 3.1.2 '@types/find-cache-dir': 3.2.1 - '@yarnpkg/esbuild-plugin-pnp': 3.0.0-rc.15(esbuild@0.17.12) + '@yarnpkg/esbuild-plugin-pnp': 3.0.0-rc.15(esbuild@0.18.20) browser-assert: 1.2.1 - ejs: 3.1.8 - esbuild: 0.17.12 + ejs: 3.1.9 + esbuild: 0.18.20 esbuild-plugin-alias: 0.2.1 express: 4.18.2 find-cache-dir: 3.3.2 - fs-extra: 11.1.0 + fs-extra: 11.1.1 process: 0.11.10 util: 0.12.5 transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/builder-vite@7.0.2(typescript@5.0.2)(vite@4.2.0): - resolution: {integrity: sha512-G6CD2Gf2zwzRslvNvqgz4FeADVEA9XA4Mw6+NM6Twc+Wy/Ah482dvHS9ApSgirtGyBKjOfdHn1xQT4Z+kzbJnw==} + /@storybook/builder-vite@7.3.2(typescript@5.0.2)(vite@4.2.0): + resolution: {integrity: sha512-9xB3Z6QfDBX6Daj+LFldhavA8O7JU2E1dL6IHfaTLIamFH884Sl5Svq3GS3oh4/EbB/GifpVKEiwlvJaINCj+A==} peerDependencies: '@preact/preset-vite': '*' - '@storybook/mdx1-csf': '>=1.0.0-next.1' typescript: '>= 4.3.x' vite: ^3.0.0 || ^4.0.0 vite-plugin-glimmerx: '*' peerDependenciesMeta: '@preact/preset-vite': optional: true - '@storybook/mdx1-csf': - optional: true typescript: optional: true vite-plugin-glimmerx: optional: true dependencies: - '@storybook/channel-postmessage': 7.0.2 - '@storybook/channel-websocket': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/csf-plugin': 7.0.2 - '@storybook/mdx2-csf': 1.0.0 - '@storybook/node-logger': 7.0.2 - '@storybook/preview': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/csf-plugin': 7.3.2 + '@storybook/mdx2-csf': 1.1.0 + '@storybook/node-logger': 7.3.2 + '@storybook/preview': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 + '@types/find-cache-dir': 3.2.1 browser-assert: 1.2.1 es-module-lexer: 0.9.3 express: 4.18.2 - fs-extra: 11.1.0 - glob: 8.1.0 - glob-promise: 6.0.2(glob@8.1.0) - magic-string: 0.27.0 + find-cache-dir: 3.3.2 + fs-extra: 11.1.1 + magic-string: 0.30.3 remark-external-links: 8.0.0 remark-slug: 6.1.0 - rollup: 3.19.1 + rollup: 3.28.1 typescript: 5.0.2 vite: 4.2.0(@types/node@18.16.3)(sass@1.59.3) transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/channel-postmessage@7.0.2: - resolution: {integrity: sha512-SZ/KqnZcx10W9hJbrzBKcP9dmgaeTaXugUhcgw1IkmjKWdsKazqFZCPwQWZZKAmhO4wYbyYOhkz3wfSIeB4mFw==} - dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/global': 5.0.0 - qs: 6.11.0 - telejson: 7.0.4 + /@storybook/channels@7.0.2: + resolution: {integrity: sha512-qkI8mFy9c8mxN2f01etayKhCaauL6RAsxRzbX1/pKj6UqhHWqqUbtHwymrv4hG5qDYjV1e9pd7ae5eNF8Kui0g==} dev: true - /@storybook/channel-websocket@7.0.2: - resolution: {integrity: sha512-YU3lFId6Nsi75ddA+3qfbnLfNUPswboYyx+SALhaLuXqz7zqfzX4ezMgxeS/h0gRlUJ7nf2/yJ5qie/kZaizjw==} + /@storybook/channels@7.3.2: + resolution: {integrity: sha512-GG5+qzv2OZAzXonqUpJR81f2pjKExj7v5MoFJhKYgb3Y+jVYlUzBHBjhQZhuQczP4si418/jvjimvU1PZ4hqcg==} dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - telejson: 7.0.4 - dev: true - - /@storybook/channels@7.0.2: - resolution: {integrity: sha512-qkI8mFy9c8mxN2f01etayKhCaauL6RAsxRzbX1/pKj6UqhHWqqUbtHwymrv4hG5qDYjV1e9pd7ae5eNF8Kui0g==} + qs: 6.11.2 + telejson: 7.2.0 + tiny-invariant: 1.3.1 dev: true - /@storybook/cli@7.0.2: - resolution: {integrity: sha512-xMM2QdXNGg09wuXzAGroKrbsnaHSFPmtmefX1XGALhHuKVwxOoC2apWMpek6gY/9vh5EIRTog2Dvfd2BzNrT6Q==} + /@storybook/cli@7.3.2: + resolution: {integrity: sha512-RnqE/6KSelL9TQ44uCIU5xvUhY9zXM2Upanr0hao72x44rvlGQbV262pHdkVIYsn0wi8QzYtnoxQPLSqUfUDfA==} hasBin: true dependencies: - '@babel/core': 7.21.3 - '@babel/preset-env': 7.20.2(@babel/core@7.21.3) - '@ndelangen/get-tarball': 3.0.7 - '@storybook/codemod': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/core-server': 7.0.2 - '@storybook/csf-tools': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/telemetry': 7.0.2 - '@storybook/types': 7.0.2 + '@babel/core': 7.22.11 + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/types': 7.22.11 + '@ndelangen/get-tarball': 3.0.9 + '@storybook/codemod': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/core-server': 7.3.2 + '@storybook/csf-tools': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/telemetry': 7.3.2 + '@storybook/types': 7.3.2 '@types/semver': 7.3.13 - boxen: 5.1.2 + '@yarnpkg/fslib': 2.10.3 + '@yarnpkg/libzip': 2.3.0 chalk: 4.1.2 commander: 6.2.1 cross-spawn: 7.0.3 detect-indent: 6.1.0 - envinfo: 7.8.1 + envinfo: 7.10.0 execa: 5.1.1 express: 4.18.2 find-up: 5.0.0 - fs-extra: 11.1.0 + fs-extra: 11.1.1 get-npm-tarball-url: 2.0.3 get-port: 5.1.1 - giget: 1.0.0 + giget: 1.1.2 globby: 11.1.0 - jscodeshift: 0.14.0(@babel/preset-env@7.20.2) + jscodeshift: 0.14.0(@babel/preset-env@7.22.10) leven: 3.1.0 + ora: 5.4.1 prettier: 2.8.4 prompts: 2.4.2 puppeteer-core: 2.1.1 read-pkg-up: 7.0.1 semver: 7.3.8 - shelljs: 0.8.5 - simple-update-notifier: 1.1.0 + simple-update-notifier: 2.0.0 strip-json-comments: 3.1.1 tempy: 1.0.1 ts-dedent: 2.2.0 @@ -5193,22 +7050,29 @@ packages: '@storybook/global': 5.0.0 dev: true - /@storybook/codemod@7.0.2: - resolution: {integrity: sha512-D9PdByxJlFiaDJcLkM+RN1DHCj4VfQIlSZkADOcNtI4o9H064oiMloWDGZiR1i1FCYMSXuWmW6tMsuCVebA+Nw==} + /@storybook/client-logger@7.3.2: + resolution: {integrity: sha512-T7q/YS5lPUE6xjz9EUwJ/v+KCd5KU9dl1MQ9RcH7IpM73EtQZeNSuM9/P96uKXZTf0wZOUBTXVlTzKr66ZB/RQ==} dependencies: - '@babel/core': 7.21.3 - '@babel/preset-env': 7.21.4(@babel/core@7.21.3) - '@babel/types': 7.21.3 - '@storybook/csf': 0.1.0 - '@storybook/csf-tools': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/global': 5.0.0 + dev: true + + /@storybook/codemod@7.3.2: + resolution: {integrity: sha512-B2P91aYhlxdk7zeQOq0VBnDox2HEcboP2unSh6Vcf4V8j2FCdPvBIM7ZkT9p15FHfyOHvvrtf56XdBIyD8/XJA==} + dependencies: + '@babel/core': 7.22.11 + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/types': 7.22.11 + '@storybook/csf': 0.1.1 + '@storybook/csf-tools': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/types': 7.3.2 + '@types/cross-spawn': 6.0.2 cross-spawn: 7.0.3 globby: 11.1.0 - jscodeshift: 0.14.0(@babel/preset-env@7.21.4) + jscodeshift: 0.14.0(@babel/preset-env@7.22.10) lodash: 4.17.21 prettier: 2.8.4 - recast: 0.23.1 + recast: 0.23.4 transitivePeerDependencies: - supports-color dev: true @@ -5220,7 +7084,7 @@ packages: react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: '@storybook/client-logger': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) '@storybook/types': 7.0.2 @@ -5231,36 +7095,64 @@ packages: util-deprecate: 1.0.2 dev: true - /@storybook/core-client@7.0.2: - resolution: {integrity: sha512-tr6Uv41YD2O0xiUrtgujiY1QxuznhbyUI0BRsSh49e8cx3QoW7FgPy7IVZHgb17DXKZ/wY/hgdyTTB87H6IbLA==} + /@storybook/components@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-hsa1OJx4yEtLHTzrCxq8G9U5MTbcTuItj9yp1gsW9RTNc/V1n/rReQv4zE/k+//2hDsLrS62o3yhZ9VksRhLNw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 + '@radix-ui/react-select': 1.2.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toolbar': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/global': 5.0.0 + '@storybook/icons': 1.1.6(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + use-resize-observer: 9.1.0(react-dom@18.2.0)(react@18.2.0) + util-deprecate: 1.0.2 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/core-common@7.0.2: - resolution: {integrity: sha512-DayFPTCj695tnEKLuDlogclBim8mzdrbj9U1xzFm23BUReheGSGdLl2zrb3mP1l9Zj4xJ/Ctst1KN9SFbW84vw==} + /@storybook/core-client@7.3.2: + resolution: {integrity: sha512-K2jCnjZiUUskFjKUj7m1FTCphIwBv0KPOE5JCd0UR7un1P1G1kdXMctADE6fHosrW73xRrad9CBSyyetUVQQOA==} dependencies: - '@storybook/node-logger': 7.0.2 - '@storybook/types': 7.0.2 - '@types/node': 16.18.11 + '@storybook/client-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + dev: true + + /@storybook/core-common@7.3.2: + resolution: {integrity: sha512-W+X7JXV0UmHuUl9xSF/xzz1+P7VM8xHt7ORfp8yrtJRwLHURqHvFFQC+NUHBKno1Ydtt/Uch7QNOWUlQKmiWEw==} + dependencies: + '@storybook/node-logger': 7.3.2 + '@storybook/types': 7.3.2 + '@types/find-cache-dir': 3.2.1 + '@types/node': 16.18.46 + '@types/node-fetch': 2.6.4 '@types/pretty-hrtime': 1.0.1 chalk: 4.1.2 - esbuild: 0.17.12 - esbuild-register: 3.4.2(esbuild@0.17.12) - file-system-cache: 2.0.2 + esbuild: 0.18.20 + esbuild-register: 3.4.2(esbuild@0.18.20) + file-system-cache: 2.3.0 + find-cache-dir: 3.3.2 find-up: 5.0.0 - fs-extra: 11.1.0 - glob: 8.1.0 - glob-promise: 6.0.2(glob@8.1.0) - handlebars: 4.7.7 + fs-extra: 11.1.1 + glob: 10.3.3 + handlebars: 4.7.8 lazy-universal-dotenv: 4.0.0 + node-fetch: 2.6.9 picomatch: 2.3.1 pkg-dir: 5.0.0 pretty-hrtime: 1.0.3 resolve-from: 5.0.0 ts-dedent: 2.2.0 transitivePeerDependencies: + - encoding - supports-color dev: true @@ -5268,51 +7160,55 @@ packages: resolution: {integrity: sha512-1DCHCwHRL3+rlvnVVc/BCfReP31XaT2WYgcLeGTmkX1E43Po1MkgcM7PnJPSaa9POvSqZ+6YLZv5Bs1SXbufow==} dev: true - /@storybook/core-server@7.0.2: - resolution: {integrity: sha512-7ipGws8YffVaiwkc+D0+MfZc/Sy52aKenG3nDJdK4Ajmp5LPAlelb/sxIhfRvoHDbDsy2FQNz++Mb55Yh03KkA==} + /@storybook/core-events@7.3.2: + resolution: {integrity: sha512-DCrM3s+sxLKS8vl0zB+1tZEtcl5XQTOGl46XgRRV/SIBabFbsC0l5pQPswWkTUsIqdREtiT0YUHcXB1+YDyFvA==} + dev: true + + /@storybook/core-server@7.3.2: + resolution: {integrity: sha512-TLMEptmfqYLu4bayRV5m8T3R50uR07Fwja1n/8CCmZOGWjnr5kXMFRkD7+hj7wm82yoidfd23bmVcRU9mlG+tg==} dependencies: - '@aw-web-design/x-default-browser': 1.4.88 + '@aw-web-design/x-default-browser': 1.4.126 '@discoveryjs/json-ext': 0.5.7 - '@storybook/builder-manager': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 - '@storybook/csf-tools': 7.0.2 + '@storybook/builder-manager': 7.3.2 + '@storybook/channels': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/csf-tools': 7.3.2 '@storybook/docs-mdx': 0.1.0 '@storybook/global': 5.0.0 - '@storybook/manager': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/telemetry': 7.0.2 - '@storybook/types': 7.0.2 - '@types/detect-port': 1.3.2 - '@types/node': 16.18.11 - '@types/node-fetch': 2.6.2 + '@storybook/manager': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/telemetry': 7.3.2 + '@storybook/types': 7.3.2 + '@types/detect-port': 1.3.3 + '@types/node': 16.18.46 '@types/pretty-hrtime': 1.0.1 '@types/semver': 7.3.13 - better-opn: 2.1.1 - boxen: 5.1.2 + better-opn: 3.0.2 chalk: 4.1.2 cli-table3: 0.6.3 compression: 1.7.4 detect-port: 1.5.1 express: 4.18.2 - fs-extra: 11.1.0 + fs-extra: 11.1.1 globby: 11.1.0 ip: 2.0.0 lodash: 4.17.21 - node-fetch: 2.6.9 - open: 8.4.0 + open: 8.4.2 pretty-hrtime: 1.0.3 prompts: 2.4.2 read-pkg-up: 7.0.1 semver: 7.3.8 serve-favicon: 2.5.0 - telejson: 7.0.4 + telejson: 7.2.0 + tiny-invariant: 1.3.1 ts-dedent: 2.2.0 + util: 0.12.5 util-deprecate: 1.0.2 watchpack: 2.4.0 - ws: 8.12.0 + ws: 8.13.0 transitivePeerDependencies: - bufferutil - encoding @@ -5320,33 +7216,33 @@ packages: - utf-8-validate dev: true - /@storybook/csf-plugin@7.0.2: - resolution: {integrity: sha512-aGuo+G6G5IwSGkmc+OUA796sOfvJMaQj8QS/Zh5F0nL4ZlQvghHpXON8cRHHvmXHQqUo07KLiy7CZh2I2oq4iQ==} + /@storybook/csf-plugin@7.3.2: + resolution: {integrity: sha512-uXJLJkRQeXnI2jHRdHfjJCbtEDohqzCrADh1xDfjqy/MQ/Sh2iFnRBCbEXsrxROBMh7Ow88/hJdy+vX0ZQh9fA==} dependencies: - '@storybook/csf-tools': 7.0.2 - unplugin: 0.10.2 + '@storybook/csf-tools': 7.3.2 + unplugin: 1.4.0 transitivePeerDependencies: - supports-color dev: true - /@storybook/csf-tools@7.0.2: - resolution: {integrity: sha512-sOp355yQSpYiMqNSopmFYWZkPPRJdGgy4tpxGGLxpOZMygK3j1wQ/WQtl2Z0h61KP0S0dl6hrs0pHQz3A/eVrw==} + /@storybook/csf-tools@7.3.2: + resolution: {integrity: sha512-54UaOsx9QZxiuMSpX01kSAEYuZYaB72Zz8ihlVrKZbIPTSJ6SYcM/jzNCGf1Rz7AjgU2UjXCSs5zBq5t37Nuqw==} dependencies: - '@babel/generator': 7.21.3 - '@babel/parser': 7.21.3 - '@babel/traverse': 7.21.3 - '@babel/types': 7.21.4 - '@storybook/csf': 0.1.0 - '@storybook/types': 7.0.2 - fs-extra: 11.1.0 - recast: 0.23.1 + '@babel/generator': 7.22.10 + '@babel/parser': 7.22.11 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + '@storybook/csf': 0.1.1 + '@storybook/types': 7.3.2 + fs-extra: 11.1.1 + recast: 0.23.4 ts-dedent: 2.2.0 transitivePeerDependencies: - supports-color dev: true - /@storybook/csf@0.1.0: - resolution: {integrity: sha512-uk+jMXCZ8t38jSTHk2o5btI+aV2Ksbvl6DoOv3r6VaCM1KZqeuMwtwywIQdflkA8/6q/dKT8z8L+g8hC4GC3VQ==} + /@storybook/csf@0.1.1: + resolution: {integrity: sha512-4hE3AlNVxR60Wc5KSC68ASYzUobjPqtSKyhV6G+ge0FIXU55N5nTY7dXGRZHQGDBPq+XqchMkIdlkHPRs8nTHg==} dependencies: type-fest: 2.19.0 dev: true @@ -5355,17 +7251,17 @@ packages: resolution: {integrity: sha512-JDaBR9lwVY4eSH5W8EGHrhODjygPd6QImRbwjAuJNEnY0Vw4ie3bPkeGfnacB3OBW6u/agqPv2aRlR46JcAQLg==} dev: true - /@storybook/docs-tools@7.0.2: - resolution: {integrity: sha512-w4D5BURrYjLbLGG9VKAaKU2dSdukszxRE3HWkJyhQU9R1JHvS3n8ntcMqYPqRfoHCOeBLBxP0edDYcAfzGNDYQ==} + /@storybook/docs-tools@7.3.2: + resolution: {integrity: sha512-MSmAiL/lg+B14CIKD6DvkBPdTDfGBSSt3bE+vW2uW9ohNJB5eWePZLQZUe34uZuunn3uqyTAgbEF7KjrtGZ/MQ==} dependencies: - '@babel/core': 7.21.3 - '@storybook/core-common': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/core-common': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 '@types/doctrine': 0.0.3 doctrine: 3.0.0 lodash: 4.17.21 transitivePeerDependencies: + - encoding - supports-color dev: true @@ -5373,20 +7269,31 @@ packages: resolution: {integrity: sha512-FcOqPAXACP0I3oJ/ws6/rrPT9WGhu915Cg8D02a9YxLo0DE9zI+a9A5gRGvmQ09fiWPukqI8ZAEoQEdWUKMQdQ==} dev: true - /@storybook/manager-api@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-PbLj9Rc5uCMPfMdaXv1wE3koA3+d0rmZ3BJI8jeq+mfZEvpvfI4OOpRioT1q04CkkVomFOVFTyO0Q/o6Rb5N7g==} + /@storybook/icons@1.1.6(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-co5gDCYPojRAc5lRMnWxbjrR1V37/rTmAo9Vok4a1hDpHZIwkGTWesdzvYivSQXYFxZTpxdM1b5K3W87brnahw==} + engines: {node: '>=14.0.0'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/manager-api@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-EEosLcc+CPLjorLf2+rGLBW0sH0SHVcB1yClLIzKM5Wt8Cl/0l19wNtGMooE/28SDLA4DPIl4WDnP83wRE1hsg==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/router': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/router': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 dequal: 2.0.3 lodash: 4.17.21 memoizerific: 1.11.3 @@ -5394,57 +7301,51 @@ packages: react-dom: 18.2.0(react@18.2.0) semver: 7.3.8 store2: 2.14.2 - telejson: 7.0.4 + telejson: 7.2.0 ts-dedent: 2.2.0 dev: true - /@storybook/manager@7.0.2: - resolution: {integrity: sha512-jsFsFKG0rPNYfuRm/WSXGMBy8vnALyFWU330ObDmfU0JID3SeLlVqAOZT1GlwI6vupYpWodsN6qPZKRmC8onRw==} + /@storybook/manager@7.3.2: + resolution: {integrity: sha512-nA3XcnD36WUjgMCtID2M4DWYZh6MnabItXvKXGbNUkI8SVaIekc5nEgeplFyqutL11eKz3Es/FwwEP+mePbWfw==} dev: true - /@storybook/mdx2-csf@1.0.0: - resolution: {integrity: sha512-dBAnEL4HfxxJmv7LdEYUoZlQbWj9APZNIbOaq0tgF8XkxiIbzqvgB0jhL/9UOrysSDbQWBiCRTu2wOVxedGfmw==} + /@storybook/mdx2-csf@1.1.0: + resolution: {integrity: sha512-TXJJd5RAKakWx4BtpwvSNdgTDkKM6RkXU8GK34S/LhidQ5Pjz3wcnqb0TxEkfhK/ztbP8nKHqXFwLfa2CYkvQw==} dev: true - /@storybook/node-logger@7.0.2: - resolution: {integrity: sha512-UENpXxB1yDqP7JXaODJo+pbGt5y3NFBNurBr4+pI4bMAC4ARjpgRE4wp6fxUKFPu9MAR10oCdcLEHkaVUAjYRg==} - dependencies: - '@types/npmlog': 4.1.4 - chalk: 4.1.2 - npmlog: 5.0.1 - pretty-hrtime: 1.0.3 + /@storybook/node-logger@7.3.2: + resolution: {integrity: sha512-XCCYiLa5mQ7KeDQcZ4awlyWDmtxJHLIJeedvXx29JUNztUjgwyon9rlNvxtxtGj6171zgn9MERFh920WyJOOOQ==} dev: true - /@storybook/postinstall@7.0.2: - resolution: {integrity: sha512-Hhiu3+N3ZDcbrhOCBJTDJbn/mC4l0v3ziyAP3yalq/2ZR9R5kfsEHHakKmswsKKV+ey0gNGijFTy3soU5oSs+A==} + /@storybook/postinstall@7.3.2: + resolution: {integrity: sha512-23/QUseeVaYjqexq4O1f1g/Fxq+pNGD+/wbXLPkdwNydutGwMZ3XAD8jcm+zeOmkbUPN8jQzKUXqO2OE/GgvHg==} dev: true - /@storybook/preview-api@7.0.2: - resolution: {integrity: sha512-QAlJM/r92+dQe/kB7MTTR9b/1mt9UJjxNjazGdEWipA/nw23kOF3o/hBcvKwBYkit4zGYsX70H+vuzW8hCo/lA==} + /@storybook/preview-api@7.3.2: + resolution: {integrity: sha512-exQrWQQLwf/nXB6OEuQScygN5iO914iNQAvicaJ7mrX9L1ypIq1PpXgJR3mSezBd9dhOMBP/BMy1Zck/wBEL9A==} dependencies: - '@storybook/channel-postmessage': 7.0.2 - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/types': 7.0.2 + '@storybook/types': 7.3.2 '@types/qs': 6.9.7 dequal: 2.0.3 lodash: 4.17.21 memoizerific: 1.11.3 - qs: 6.11.0 - synchronous-promise: 2.0.16 + qs: 6.11.2 + synchronous-promise: 2.0.17 ts-dedent: 2.2.0 util-deprecate: 1.0.2 dev: true - /@storybook/preview@7.0.2: - resolution: {integrity: sha512-U7MZkDT9bBq7HggLAXmTO9gI4eqhYs26fZS0L6iTE/PCX4Wg2TJBJSq2X8jhDXRqJFOt8SrQ756+V5Vtwrh4Og==} + /@storybook/preview@7.3.2: + resolution: {integrity: sha512-UXgImhD7xa+nYgXRcNFQdTqQT1725mOzWbQUtYPMJXkHO+t251hQrEc81tMzSSPEgPrFY8wndpEqTt8glFm91g==} dev: true - /@storybook/react-dom-shim@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-fMl0aV7mJ3wyQKvt6z+rZuiIiSd9YinS77IJ1ETHqVZ4SxWriOS0GFKP6sZflrlpShoZBh+zl1lDPG7ZZdrQGw==} + /@storybook/react-dom-shim@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-63ysybmpl9UULmLu/aUwWwhjf5QEWTvnMW9r8Z3LF3sW8Z698ZsssdThzNWqw0zlwTlgnQA4ta2Df4/oVXR0+Q==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5453,8 +7354,8 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true - /@storybook/react-vite@7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2)(vite@4.2.0): - resolution: {integrity: sha512-1bDrmGo6imxBzZKJJ+SEHPuDn474JY3Yatm0cPaNVtlYhbnbiTPa3PxhI4U3233l4Qsc6DXNLKvi++j/knXDCw==} + /@storybook/react-vite@7.3.2(react-dom@18.2.0)(react@18.2.0)(rollup@2.79.1)(typescript@5.0.2)(vite@4.2.0): + resolution: {integrity: sha512-lfDrcESQkrqVh1PkFgiJMrfhGYNckQ3rB2ZCvvzruHYWe/9B40EbMxGZauLg7B7M5j2Rzj+sa7Jfr3dasm+GJA==} engines: {node: '>=16'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5462,26 +7363,27 @@ packages: vite: ^3.0.0 || ^4.0.0 dependencies: '@joshwooding/vite-plugin-react-docgen-typescript': 0.2.1(typescript@5.0.2)(vite@4.2.0) - '@rollup/pluginutils': 4.2.1 - '@storybook/builder-vite': 7.0.2(typescript@5.0.2)(vite@4.2.0) - '@storybook/react': 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) + '@rollup/pluginutils': 5.0.4(rollup@2.79.1) + '@storybook/builder-vite': 7.3.2(typescript@5.0.2)(vite@4.2.0) + '@storybook/react': 7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) '@vitejs/plugin-react': 3.1.0(vite@4.2.0) ast-types: 0.14.2 - magic-string: 0.27.0 + magic-string: 0.30.3 react: 18.2.0 react-docgen: 6.0.0-alpha.3 react-dom: 18.2.0(react@18.2.0) vite: 4.2.0(@types/node@18.16.3)(sass@1.59.3) transitivePeerDependencies: - '@preact/preset-vite' - - '@storybook/mdx1-csf' + - encoding + - rollup - supports-color - typescript - vite-plugin-glimmerx dev: true - /@storybook/react@7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2): - resolution: {integrity: sha512-2P7Oju1XKWMyn75dO0vjL4gthzBL/lLiCBRyAHKXZJ1H2eNdWjXkOOtH1HxnbRcXjWSU4tW96dqKY8m0iR9zAA==} + /@storybook/react@7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2): + resolution: {integrity: sha512-VMXy+soLnEW+lN1sfkkMGkmk3gnS3KLfEk0JssSlj+jGA4cPpvO+P1uGNkN8MjdiU9VaWt0aZ7uRdwx0rrfFUw==} engines: {node: '>=16.0.0'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5491,21 +7393,21 @@ packages: typescript: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-client': 7.0.2 - '@storybook/docs-tools': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-client': 7.3.2 + '@storybook/docs-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/preview-api': 7.0.2 - '@storybook/react-dom-shim': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/preview-api': 7.3.2 + '@storybook/react-dom-shim': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 '@types/escodegen': 0.0.6 '@types/estree': 0.0.51 - '@types/node': 16.18.11 + '@types/node': 16.18.46 acorn: 7.4.1 acorn-jsx: 5.3.2(acorn@7.4.1) acorn-walk: 7.2.0 - escodegen: 2.0.0 - html-tags: 3.2.0 + escodegen: 2.1.0 + html-tags: 3.3.1 lodash: 4.17.21 prop-types: 15.8.1 react: 18.2.0 @@ -5516,33 +7418,33 @@ packages: typescript: 5.0.2 util-deprecate: 1.0.2 transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/router@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-ZB2vucfayZUrMLBlXju4v6CNOQQb0YKDLw5RoojdBxOsUFtnp5UiPOE+I8PQR63EBwnRjozeibV1XSM+GlQb5w==} + /@storybook/router@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-J3QPudwCJhdnfqPx9GaNDlnsjJ6JbFta/ypp3EkHntyuuaNBeNP3Aq73DJJY2XMTS2Xdw8tD9Y9Y9gCFHJXMDQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/client-logger': 7.0.2 + '@storybook/client-logger': 7.3.2 memoizerific: 1.11.3 - qs: 6.11.0 + qs: 6.11.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) dev: true - /@storybook/telemetry@7.0.2: - resolution: {integrity: sha512-s2PIwI9nVYQBf3h40EFHLynYUfdqzRJMXyaCWJdVQuvdQfRkAn3CLXaubK+VdjC869z3ZfW20EMu3Mbgzcc0HA==} + /@storybook/telemetry@7.3.2: + resolution: {integrity: sha512-BmgwaZGoR2ZzGZpcO5ipc4uMd9y28qmu9Ynx054Q3mb86daJrw4CU18TVi5UoFa9qmygQhoHx2gaK2QStNtqCg==} dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-common': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/csf-tools': 7.3.2 chalk: 4.1.2 detect-package-manager: 2.0.1 - fetch-retry: 5.0.3 - fs-extra: 11.1.0 - isomorphic-unfetch: 3.1.0 - nanoid: 3.3.4 + fetch-retry: 5.0.6 + fs-extra: 11.1.1 read-pkg-up: 7.0.1 transitivePeerDependencies: - encoding @@ -5555,7 +7457,7 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@emotion/use-insertion-effect-with-fallbacks': 1.0.0(react@18.2.0) + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) '@storybook/client-logger': 7.0.2 '@storybook/global': 5.0.0 memoizerific: 1.11.3 @@ -5563,6 +7465,20 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true + /@storybook/theming@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-npVsnmNAtqGwl1K7vLC/hcVhL8tBC8G0vdZXEcufF0jHdQmRCUs9ZVrnR6W0LCrtmIHDaDoO7PqJVSzu2wgVxw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/global': 5.0.0 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + /@storybook/types@7.0.2: resolution: {integrity: sha512-0OCt/kAexa8MCcljxA+yZxGMn0n2U2Ync0KxotItqNbKBKVkaLQUls0+IXTWSCpC/QJvNZ049jxUHHanNi/96w==} dependencies: @@ -5572,6 +7488,15 @@ packages: file-system-cache: 2.0.2 dev: true + /@storybook/types@7.3.2: + resolution: {integrity: sha512-1UHC1r2J6H9dEpj4pp9a16P1rTL87V9Yc6TtYBpp7m+cxzyIZBRvu1wZFKmRB51RXE/uDaxGRKzfNRfgTALcIQ==} + dependencies: + '@storybook/channels': 7.3.2 + '@types/babel__core': 7.20.1 + '@types/express': 4.17.17 + file-system-cache: 2.3.0 + dev: true + /@svgr/babel-plugin-add-jsx-attribute@6.5.1(@babel/core@7.21.3): resolution: {integrity: sha512-9PYGcXrAxitycIjRmZB+Q0JaN07GZIWaTBIGQzfaZv+qr1n8X1XUEJ5rZ/vx6OVD9RRYlrNnXWExQXcmZeD/BQ==} engines: {node: '>=10'} @@ -5913,12 +7838,22 @@ packages: '@types/node': 18.16.3 dev: true + /@types/byte-size@8.1.0: + resolution: {integrity: sha512-LCIlZh8vyx+I2fgRycE1D34c33QDppYY6quBYYoaOpQ1nGhJ/avSP2VlrAefVotjJxgSk6WkKo0rTcCJwGG7vA==} + dev: true + /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: '@types/node': 18.16.3 dev: true + /@types/cross-spawn@6.0.2: + resolution: {integrity: sha512-KuwNhp3eza+Rhu8IFI5HUXRP0LIhqH5cAjubUvGXXthh4YYBuP2ntwEX+Cz8GJoZUHlKo247wPWOfA9LYEq4cw==} + dependencies: + '@types/node': 18.16.3 + dev: true + /@types/d3-array@3.0.4: resolution: {integrity: sha512-nwvEkG9vYOc0Ic7G7kwgviY4AQlTfYGIZ0fqB7CQHXGyYM6nO7kJh5EguSNA3jfh4rq7Sb7eMVq8isuvg2/miQ==} dev: false @@ -5967,8 +7902,8 @@ packages: '@types/ms': 0.7.31 dev: false - /@types/detect-port@1.3.2: - resolution: {integrity: sha512-xxgAGA2SAU4111QefXPSp5eGbDm/hW6zhvYl9IeEPZEry9F4d66QAHm5qpUXjb6IsevZV/7emAEx5MhP6O192g==} + /@types/detect-port@1.3.3: + resolution: {integrity: sha512-bV/jQlAJ/nPY3XqSatkGpu+nGzou+uSwrH1cROhn+jBFg47yaNH+blW4C7p9KhopC7QxCv/6M86s37k8dMk0Yg==} dev: true /@types/diff@5.0.2: @@ -5979,8 +7914,12 @@ packages: resolution: {integrity: sha512-w5jZ0ee+HaPOaX25X2/2oGR/7rgAQSYII7X7pp0m9KgBfMP7uKfMfTvcpl5Dj+eDBbpxKGiqE+flqDr6XTd2RA==} dev: true - /@types/ejs@3.1.1: - resolution: {integrity: sha512-RQul5wEfY7BjWm0sYY86cmUN/pcXWGyVxWX93DFFJvcrxax5zKlieLwA3T77xJGwNcZW0YW6CYG70p1m8xPFmA==} + /@types/ejs@3.1.2: + resolution: {integrity: sha512-ZmiaE3wglXVWBM9fyVC17aGPkLo/UgaOjEiI2FXQfyczrCefORPxIe+2dVmnmk3zkVIbizjrlQzmPGhSYGXG5g==} + dev: true + + /@types/emscripten@1.39.7: + resolution: {integrity: sha512-tLqYV94vuqDrXh515F/FOGtBcRMTPGvVV1LzLbtYDcQmmhtpf/gLYf+hikBbQk8MzOHNz37wpFfJbYAuSn8HqA==} dev: true /@types/es-aggregate-error@1.0.2: @@ -6017,6 +7956,15 @@ packages: '@types/range-parser': 1.2.4 dev: true + /@types/express-serve-static-core@4.17.36: + resolution: {integrity: sha512-zbivROJ0ZqLAtMzgzIUC4oNqDG9iF0lSsAqpOD9kbs5xcIM3dTiyuHvBc7R8MtWBp3AAWGaovJa+wzWPjLYW7Q==} + dependencies: + '@types/node': 18.16.3 + '@types/qs': 6.9.7 + '@types/range-parser': 1.2.4 + '@types/send': 0.17.1 + dev: true + /@types/express@4.17.16: resolution: {integrity: sha512-LkKpqRZ7zqXJuvoELakaFYuETHjZkSol8EV6cNnyishutDBCCdv6+dsKPbKkCcIk57qRphOLY5sEgClw1bO3gA==} dependencies: @@ -6026,6 +7974,15 @@ packages: '@types/serve-static': 1.15.0 dev: true + /@types/express@4.17.17: + resolution: {integrity: sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==} + dependencies: + '@types/body-parser': 1.19.2 + '@types/express-serve-static-core': 4.17.36 + '@types/qs': 6.9.7 + '@types/serve-static': 1.15.2 + dev: true + /@types/find-cache-dir@3.2.1: resolution: {integrity: sha512-frsJrz2t/CeGifcu/6uRo4b+SzAwT4NYCVPu1GN8IB9XTzrpPkGuV0tmh9mN+/L0PklAlsC3u5Fxt0ju00LXIw==} dev: true @@ -6041,13 +7998,6 @@ packages: '@types/node': 18.16.3 dev: true - /@types/glob@8.1.0: - resolution: {integrity: sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==} - dependencies: - '@types/minimatch': 5.1.2 - '@types/node': 18.16.3 - dev: true - /@types/graceful-fs@4.1.6: resolution: {integrity: sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==} dependencies: @@ -6066,6 +8016,10 @@ packages: '@types/react': 18.2.20 hoist-non-react-statics: 3.3.2 + /@types/http-errors@2.0.1: + resolution: {integrity: sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==} + dev: true + /@types/istanbul-lib-coverage@2.0.4: resolution: {integrity: sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==} dev: true @@ -6130,14 +8084,18 @@ packages: resolution: {integrity: sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==} dev: false - /@types/mdx@2.0.3: - resolution: {integrity: sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==} + /@types/mdx@2.0.7: + resolution: {integrity: sha512-BG4tyr+4amr3WsSEmHn/fXPqaCba/AYZ7dsaQTiavihQunHSIxk+uAtqsjvicNpyHN6cm+B9RVrUOtW9VzIKHw==} dev: true /@types/mime-types@2.1.1: resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==} dev: true + /@types/mime@1.3.2: + resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} + dev: true + /@types/mime@3.0.1: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} dev: true @@ -6159,13 +8117,21 @@ packages: dependencies: '@types/node': 18.16.3 form-data: 3.0.1 + dev: false + + /@types/node-fetch@2.6.4: + resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} + dependencies: + '@types/node': 18.16.3 + form-data: 3.0.1 + dev: true /@types/node@14.18.43: resolution: {integrity: sha512-n3eFEaoem0WNwLux+k272P0+aq++5o05bA9CfiwKPdYPB5ZambWKdWoeHy7/OJiizMhzg27NLaZ6uzjLTzXceQ==} dev: true - /@types/node@16.18.11: - resolution: {integrity: sha512-3oJbGBUWuS6ahSnEq1eN2XrCyf4YsWI8OyCvo7c64zQJNplk3mO84t53o8lfTk+2ji59g5ycfc6qQ3fdHliHuA==} + /@types/node@16.18.46: + resolution: {integrity: sha512-Mnq3O9Xz52exs3mlxMcQuA7/9VFe/dXcrgAyfjLkABIqxXKOgBRjyazTxUbjsxDa4BP7hhPliyjVTP9RDP14xg==} dev: true /@types/node@18.16.3: @@ -6175,10 +8141,6 @@ packages: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} dev: true - /@types/npmlog@4.1.4: - resolution: {integrity: sha512-WKG4gTr8przEZBiJ5r3s8ZIAoMXNbOgQ+j/d5O4X3x6kZJRLNvyUJuUK/KoG3+8BaOHPhp2m7WC6JKKeovDSzQ==} - dev: true - /@types/parse-json@4.0.0: resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==} @@ -6305,6 +8267,13 @@ packages: /@types/semver@7.3.13: resolution: {integrity: sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==} + /@types/send@0.17.1: + resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==} + dependencies: + '@types/mime': 1.3.2 + '@types/node': 18.16.3 + dev: true + /@types/serve-static@1.15.0: resolution: {integrity: sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==} dependencies: @@ -6312,6 +8281,14 @@ packages: '@types/node': 18.16.3 dev: true + /@types/serve-static@1.15.2: + resolution: {integrity: sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==} + dependencies: + '@types/http-errors': 2.0.1 + '@types/mime': 3.0.1 + '@types/node': 18.16.3 + dev: true + /@types/sinonjs__fake-timers@8.1.1: resolution: {integrity: sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==} dev: true @@ -6615,14 +8592,30 @@ packages: resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} dev: false - /@yarnpkg/esbuild-plugin-pnp@3.0.0-rc.15(esbuild@0.17.12): + /@yarnpkg/esbuild-plugin-pnp@3.0.0-rc.15(esbuild@0.18.20): resolution: {integrity: sha512-kYzDJO5CA9sy+on/s2aIW0411AklfCi8Ck/4QDivOqsMKpStZA2SsR+X27VTggGwpStWaLrjJcDcdDMowtG8MA==} engines: {node: '>=14.15.0'} peerDependencies: esbuild: '>=0.10.0' dependencies: - esbuild: 0.17.12 - tslib: 2.5.2 + esbuild: 0.18.20 + tslib: 2.6.2 + dev: true + + /@yarnpkg/fslib@2.10.3: + resolution: {integrity: sha512-41H+Ga78xT9sHvWLlFOZLIhtU6mTGZ20pZ29EiZa97vnxdohJD2AF42rCoAoWfqUz486xY6fhjMH+DYEM9r14A==} + engines: {node: '>=12 <14 || 14.2 - 14.9 || >14.10.0'} + dependencies: + '@yarnpkg/libzip': 2.3.0 + tslib: 1.14.1 + dev: true + + /@yarnpkg/libzip@2.3.0: + resolution: {integrity: sha512-6xm38yGVIa6mKm/DUCF2zFFJhERh/QWp1ufm4cNUvxsONBmfPg8uZ9pZBdOmF6qFGr/HlT6ABBkCSx/dlEtvWg==} + engines: {node: '>=12 <14 || 14.2 - 14.9 || >14.10.0'} + dependencies: + '@types/emscripten': 1.39.7 + tslib: 1.14.1 dev: true /abab@2.0.6: @@ -6687,6 +8680,12 @@ packages: hasBin: true dev: true + /acorn@8.10.0: + resolution: {integrity: sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: true + /acorn@8.8.1: resolution: {integrity: sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==} engines: {node: '>=0.4.0'} @@ -6778,12 +8777,6 @@ packages: resolution: {integrity: sha512-nqLm4HxOTpeLOxcmB3QWmV5TcDFhW9y/fyQ+hivtDFcK4OQ+pQ5fzPnXHM1Mfcm0VkLtvVi1TCPr++Qy0Q/3EQ==} dev: false - /ansi-align@3.0.1: - resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} - dependencies: - string-width: 4.2.3 - dev: true - /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -6842,22 +8835,10 @@ packages: resolution: {integrity: sha512-jlpIfsOoNoafl92Sz//64uQHGSyMrD2vYG5d8o2a4qGvyNCvXur7bzIsWtAC/6flI2RYAp3kv8rsfBtaLm7w0g==} dev: true - /aproba@2.0.0: - resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} - dev: true - /arch@2.2.0: resolution: {integrity: sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==} dev: true - /are-we-there-yet@2.0.0: - resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==} - engines: {node: '>=10'} - dependencies: - delegates: 1.0.0 - readable-stream: 3.6.0 - dev: true - /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} dev: true @@ -6880,7 +8861,6 @@ packages: engines: {node: '>=10'} dependencies: tslib: 2.5.2 - dev: false /aria-query@5.1.3: resolution: {integrity: sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==} @@ -7004,21 +8984,21 @@ packages: resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.14.2: resolution: {integrity: sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.15.2: resolution: {integrity: sha512-c27loCv9QkZinsa5ProX751khO9DJl/AcB5c2KNtA6NRvHKS0PgLfcftz72KVq504vB0Gku5s2kUZzDBvQWvHg==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.16.1: @@ -7175,6 +9155,19 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-corejs2@0.4.5(@babel/core@7.22.11): + resolution: {integrity: sha512-19hwUH5FKl49JEsvyTcoHakh6BE0wgXLLptIyKZ3PijHc/Ci521wygORCUCCred+E/twuqRyAkE02BAWPmsHOg==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-polyfill-corejs3@0.6.0(@babel/core@7.21.3): resolution: {integrity: sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==} peerDependencies: @@ -7187,6 +9180,18 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-corejs3@0.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-z41XaniZL26WLrvjy7soabMXrfPWARN25PZoriDEiLMxAp50AUW3t35BGQUMg5xK3UrpVTtagIDklxYa+MhiNA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + core-js-compat: 3.32.1 + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-polyfill-regenerator@0.4.1(@babel/core@7.21.3): resolution: {integrity: sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==} peerDependencies: @@ -7198,6 +9203,17 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-regenerator@0.5.2(@babel/core@7.22.11): + resolution: {integrity: sha512-tAlOptU0Xj34V1Y2PNTL4Y0FOJMDB6bZmoW39FeCQIhigGLkqu3Fj6uiXpxIf6Ij274ENdYx64y6Au+ZKlb1IA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-styled-components@2.0.7(styled-components@5.3.9): resolution: {integrity: sha512-i7YhvPgVqRKfoQ66toiZ06jPNA3p6ierpfUuEWxNF+fV27Uv5gxBkf8KZLHUCc1nFA9j6+80pYoIpqCeyW3/bA==} peerDependencies: @@ -7299,11 +9315,11 @@ packages: tweetnacl: 0.14.5 dev: true - /better-opn@2.1.1: - resolution: {integrity: sha512-kIPXZS5qwyKiX/HcRvDYfmBQUa8XP17I0mYZZ0y4UhpYOSvtsLHDYqmomS+Mj20aDvD3knEiQ0ecQy2nhio3yA==} - engines: {node: '>8.0.0'} + /better-opn@3.0.2: + resolution: {integrity: sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==} + engines: {node: '>=12.0.0'} dependencies: - open: 7.4.2 + open: 8.4.2 dev: true /big-integer@1.6.51: @@ -7320,7 +9336,7 @@ packages: dependencies: buffer: 5.7.1 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 dev: true /blob-util@2.0.2: @@ -7351,20 +9367,6 @@ packages: - supports-color dev: true - /boxen@5.1.2: - resolution: {integrity: sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ==} - engines: {node: '>=10'} - dependencies: - ansi-align: 3.0.1 - camelcase: 6.3.0 - chalk: 4.1.2 - cli-boxes: 2.2.1 - string-width: 4.2.3 - type-fest: 0.20.2 - widest-line: 3.1.0 - wrap-ansi: 7.0.0 - dev: true - /bplist-parser@0.2.0: resolution: {integrity: sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw==} engines: {node: '>= 5.10.0'} @@ -7401,6 +9403,17 @@ packages: pako: 0.2.9 dev: true + /browserslist@4.21.10: + resolution: {integrity: sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + dependencies: + caniuse-lite: 1.0.30001524 + electron-to-chromium: 1.4.503 + node-releases: 2.0.13 + update-browserslist-db: 1.0.11(browserslist@4.21.10) + dev: true + /browserslist@4.21.4: resolution: {integrity: sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -7441,6 +9454,11 @@ packages: resolution: {integrity: sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==} dev: true + /byte-size@8.1.1: + resolution: {integrity: sha512-tUkzZWK0M/qdoLEqikxBWe4kumyuwjl3HO6zHTr4yEI23EojPtLYXdG1+AQY7MN0cGyNDvEaJ8wiYQm6P2bPxg==} + engines: {node: '>=12.17'} + dev: false + /bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} @@ -7451,8 +9469,8 @@ packages: engines: {node: '>= 0.8'} dev: true - /c8@7.12.0: - resolution: {integrity: sha512-CtgQrHOkyxr5koX1wEUmN/5cfDa2ckbHRA4Gy5LAL0zaCFtVWJS5++n+w4/sr2GWGerBxgTjpKeDclk/Qk6W/A==} + /c8@7.14.0: + resolution: {integrity: sha512-i04rtkkcNcCf7zsQcSv/T9EbUn4RXQ6mropeMcjFOsQXQ0iGLAr/xT6TImQg4+U9hmNpN9XdvPkjUL1IzbgxJw==} engines: {node: '>=10.12.0'} hasBin: true dependencies: @@ -7461,8 +9479,8 @@ packages: find-up: 5.0.0 foreground-child: 2.0.0 istanbul-lib-coverage: 3.2.0 - istanbul-lib-report: 3.0.0 - istanbul-reports: 3.1.5 + istanbul-lib-report: 3.0.1 + istanbul-reports: 3.1.6 rimraf: 3.0.2 test-exclude: 6.0.0 v8-to-istanbul: 9.1.0 @@ -7521,6 +9539,10 @@ packages: /caniuse-lite@1.0.30001445: resolution: {integrity: sha512-8sdQIdMztYmzfTMO6KfLny878Ln9c2M0fc7EH60IjlP4Dc4PiCy7K2Vl3ITmWgOyPgVQKa5x+UP/KqFsxj4mBg==} + /caniuse-lite@1.0.30001524: + resolution: {integrity: sha512-Jj917pJtYg9HSJBF95HVX3Cdr89JUyLT4IZ8SvM5aDRni95swKgYi3TgYLH5hnGfPE/U1dg6IfZ50UsIlLkwSA==} + dev: true + /caseless@0.12.0: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} dev: true @@ -7591,7 +9613,7 @@ packages: normalize-path: 3.0.0 readdirp: 3.6.0 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} @@ -7620,11 +9642,6 @@ packages: engines: {node: '>=6'} dev: true - /cli-boxes@2.2.1: - resolution: {integrity: sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==} - engines: {node: '>=6'} - dev: true - /cli-cursor@3.1.0: resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} engines: {node: '>=8'} @@ -7632,6 +9649,11 @@ packages: restore-cursor: 3.1.0 dev: true + /cli-spinners@2.9.0: + resolution: {integrity: sha512-4/aL9X3Wh0yiMQlE+eeRhWP6vclO3QRtw1JHKIT0FFUs5FjpFmESqtMvYZ0+lbzBw900b95mS0hohy+qn2VK/g==} + engines: {node: '>=6'} + dev: true + /cli-table3@0.6.3: resolution: {integrity: sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==} engines: {node: 10.* || >= 12.*} @@ -7722,11 +9744,6 @@ packages: /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - /color-support@1.1.3: - resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} - hasBin: true - dev: true - /colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} dev: true @@ -7735,6 +9752,10 @@ packages: resolution: {integrity: sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==} dev: true + /colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + dev: true + /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -7809,7 +9830,7 @@ packages: dependencies: buffer-from: 1.1.2 inherits: 2.0.4 - readable-stream: 2.3.7 + readable-stream: 2.3.8 typedarray: 0.0.6 dev: true @@ -7817,10 +9838,6 @@ packages: resolution: {integrity: sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==} dev: true - /console-control-strings@1.1.0: - resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} - dev: true - /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -7874,6 +9891,12 @@ packages: browserslist: 4.21.4 dev: true + /core-js-compat@3.32.1: + resolution: {integrity: sha512-GSvKDv4wE0bPnQtjklV101juQ85g6H3rm5PDP20mqlS5j0kXF3pP97YvAu5hl+uFHqMictp3b2VxOHljWMAtuA==} + dependencies: + browserslist: 4.21.10 + dev: true + /core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} dev: true @@ -8365,10 +10388,6 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - /delegates@1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - dev: true - /depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -8398,6 +10417,10 @@ packages: engines: {node: '>=8'} dev: true + /detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + dev: true + /detect-package-manager@2.0.1: resolution: {integrity: sha512-j/lJHyoLlWi6G1LDdLgvUtz60Zo5GEj+sVYtTVXnYLDPuzgC3llMxonXym9zIwhhUII8vjdw0LXxavpLqTbl1A==} engines: {node: '>=12'} @@ -8524,7 +10547,7 @@ packages: dependencies: end-of-stream: 1.4.4 inherits: 2.0.4 - readable-stream: 2.3.7 + readable-stream: 2.3.8 stream-shift: 1.0.1 dev: true @@ -8543,17 +10566,21 @@ packages: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} dev: true - /ejs@3.1.8: - resolution: {integrity: sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==} + /ejs@3.1.9: + resolution: {integrity: sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==} engines: {node: '>=0.10.0'} hasBin: true dependencies: - jake: 10.8.5 + jake: 10.8.7 dev: true /electron-to-chromium@1.4.284: resolution: {integrity: sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==} + /electron-to-chromium@1.4.503: + resolution: {integrity: sha512-LF2IQit4B0VrUHFeQkWhZm97KuJSGF2WJqq1InpY+ECpFRkXd8yTIaTtJxsO0OKDmiBYwWqcrNaXOurn2T2wiA==} + dev: true + /emittery@0.13.1: resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==} engines: {node: '>=12'} @@ -8585,10 +10612,6 @@ packages: tapable: 2.2.1 dev: true - /enquire.js@2.1.6: - resolution: {integrity: sha512-/KujNpO+PT63F7Hlpu4h3pE3TokKRHN26JYmQpPyjkRD/N57R7bPDNojMXdi7uveAKjYB7yQnartCxZnFWr0Xw==} - dev: false - /enquirer@2.3.6: resolution: {integrity: sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==} engines: {node: '>=8.6'} @@ -8600,8 +10623,8 @@ packages: resolution: {integrity: sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==} engines: {node: '>=0.12'} - /envinfo@7.8.1: - resolution: {integrity: sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==} + /envinfo@7.10.0: + resolution: {integrity: sha512-ZtUjZO6l5mwTHvc1L9+1q5p/R3wTopcfqMW8r5t8SJSKqeVI/LtajORwRFEKpEFuekjD0VBjwu1HMxL4UalIRw==} engines: {node: '>=4'} hasBin: true dev: true @@ -8920,13 +10943,13 @@ packages: resolution: {integrity: sha512-jyfL/pwPqaFXyKnj8lP8iLk6Z0m099uXR45aSN8Av1XD4vhvQutxxPzgA2bTcAwQpa1zCXDcWOlhFgyP3GKqhQ==} dev: true - /esbuild-register@3.4.2(esbuild@0.17.12): + /esbuild-register@3.4.2(esbuild@0.18.20): resolution: {integrity: sha512-kG/XyTDyz6+YDuyfB9ZoSIOOmgyFCH+xPRtsCa8W85HLRV5Csp+o3jWVbOSHgSLfyLc5DmP+KFDNwty4mEjC+Q==} peerDependencies: esbuild: '>=0.12 <1' dependencies: debug: 4.3.4(supports-color@5.5.0) - esbuild: 0.17.12 + esbuild: 0.18.20 transitivePeerDependencies: - supports-color dev: true @@ -9027,6 +11050,36 @@ packages: '@esbuild/win32-x64': 0.17.12 dev: true + /esbuild@0.18.20: + resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@esbuild/android-arm': 0.18.20 + '@esbuild/android-arm64': 0.18.20 + '@esbuild/android-x64': 0.18.20 + '@esbuild/darwin-arm64': 0.18.20 + '@esbuild/darwin-x64': 0.18.20 + '@esbuild/freebsd-arm64': 0.18.20 + '@esbuild/freebsd-x64': 0.18.20 + '@esbuild/linux-arm': 0.18.20 + '@esbuild/linux-arm64': 0.18.20 + '@esbuild/linux-ia32': 0.18.20 + '@esbuild/linux-loong64': 0.18.20 + '@esbuild/linux-mips64el': 0.18.20 + '@esbuild/linux-ppc64': 0.18.20 + '@esbuild/linux-riscv64': 0.18.20 + '@esbuild/linux-s390x': 0.18.20 + '@esbuild/linux-x64': 0.18.20 + '@esbuild/netbsd-x64': 0.18.20 + '@esbuild/openbsd-x64': 0.18.20 + '@esbuild/sunos-x64': 0.18.20 + '@esbuild/win32-arm64': 0.18.20 + '@esbuild/win32-ia32': 0.18.20 + '@esbuild/win32-x64': 0.18.20 + dev: true + /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -9079,6 +11132,18 @@ packages: source-map: 0.6.1 dev: true + /escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + dev: true + /eslint-config-prettier@8.7.0(eslint@8.36.0): resolution: {integrity: sha512-HHVXLSlVUhMSmyW4ZzEuvjpwqamgmlfkutD53cYXLikh4pt/modINRcCIApJ84czDxM4GZInwUrromsDdTImTA==} hasBin: true @@ -9542,9 +11607,9 @@ packages: resolution: {integrity: sha512-YNF+mZ/Wu2FU/gvmzuWtYc8rloubL7wfXCTgouFrnjGVXPA/EeYYA7pupXWrb3Iv1cTBeSSxxJIbK23l4MRNqg==} engines: {node: '>=8.3.0'} dependencies: - '@babel/traverse': 7.21.3 - '@babel/types': 7.21.3 - c8: 7.12.0 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + c8: 7.14.0 transitivePeerDependencies: - supports-color dev: true @@ -9823,8 +11888,8 @@ packages: resolution: {integrity: sha512-qu4mXWf4wus4idBIN/kVH+XSer8IZ9CwHP+Pd7DL7TuKNC1hP7ykon4kkBjwJF3EMX2WsFp4hH7gU7CyL7ucXw==} dev: false - /fetch-retry@5.0.3: - resolution: {integrity: sha512-uJQyMrX5IJZkhoEUBQ3EjxkeiZkppBd5jS/fMTJmfZxLSiaQjv2zD0kTvuvkSH89uFvgSlB6ueGpjD3HWN7Bxw==} + /fetch-retry@5.0.6: + resolution: {integrity: sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==} dev: true /figures@3.2.0: @@ -9848,6 +11913,13 @@ packages: ramda: 0.28.0 dev: true + /file-system-cache@2.3.0: + resolution: {integrity: sha512-l4DMNdsIPsVnKrgEXbJwDJsA5mB8rGwHYERMgqQx/xAUtChPJMre1bXBzDEqqVbWv9AIbFezXMxeEkZDSrXUOQ==} + dependencies: + fs-extra: 11.1.1 + ramda: 0.29.0 + dev: true + /file-uri-to-path@2.0.0: resolution: {integrity: sha512-hjPFI8oE/2iQPVe4gbrJ73Pp+Xfub2+WI2LlXDbsaJBwT5wuMh35WNWVYYTpnz895shtwfyutMFLFywpQAFdLg==} engines: {node: '>= 6'} @@ -9980,8 +12052,8 @@ packages: resolution: {integrity: sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==} dev: true - /flow-parser@0.198.2: - resolution: {integrity: sha512-tCQzqXbRAz0ZadIhAXGwdp/xsusADo8IK9idgc/2qCK5RmazbKDGedyykfRtzWgy7Klt4f4NZxq0o/wFUg6plQ==} + /flow-parser@0.215.1: + resolution: {integrity: sha512-qq3rdRToqwesrddyXf+Ml8Tuf7TdoJS+EMbJgC6fHAVoBCXjb4mHelNd3J+jD8ts0bSHX81FG3LN7Qn/dcl6pA==} engines: {node: '>=0.4.0'} dev: true @@ -10011,6 +12083,14 @@ packages: signal-exit: 3.0.7 dev: true + /foreground-child@3.1.1: + resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} + engines: {node: '>=14'} + dependencies: + cross-spawn: 7.0.3 + signal-exit: 4.1.0 + dev: true + /forever-agent@0.6.1: resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} dev: true @@ -10125,6 +12205,15 @@ packages: universalify: 2.0.0 dev: true + /fs-extra@11.1.1: + resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.0 + dev: true + /fs-extra@8.1.0: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} @@ -10155,8 +12244,8 @@ packages: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true - /fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -10187,21 +12276,6 @@ packages: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} dev: true - /gauge@3.0.2: - resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==} - engines: {node: '>=10'} - dependencies: - aproba: 2.0.0 - color-support: 1.1.3 - console-control-strings: 1.1.0 - has-unicode: 2.0.1 - object-assign: 4.1.1 - signal-exit: 3.0.7 - string-width: 4.2.3 - strip-ansi: 6.0.1 - wide-align: 1.1.5 - dev: true - /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -10231,6 +12305,11 @@ packages: resolution: {integrity: sha512-2MSPMu7S1iOTL+BOa6K1S62hB2zUAYNF/lV0gSVlOaacd087lc6nR1H1r0e3B1CerTo+RceOmi1iJW+vp21xcQ==} dev: false + /get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + dev: true + /get-npm-tarball-url@2.0.3: resolution: {integrity: sha512-R/PW6RqyaBQNWYaSyfrh54/qtcnOp22FHCCiRhSSZj0FP3KQWCsxxt0DzIdVTbwTqe9CtQfvl/FPD4UIPt4pqw==} engines: {node: '>=12.17'} @@ -10303,16 +12382,16 @@ packages: assert-plus: 1.0.0 dev: true - /giget@1.0.0: - resolution: {integrity: sha512-KWELZn3Nxq5+0So485poHrFriK9Bn3V/x9y+wgqrHkbmnGbjfLmZ685/SVA/ovW+ewoqW0gVI47pI4yW/VNobQ==} + /giget@1.1.2: + resolution: {integrity: sha512-HsLoS07HiQ5oqvObOI+Qb2tyZH4Gj5nYGfF9qQcZNrPw+uEFhdXtgJr01aO2pWadGHucajYDLxxbtQkm97ON2A==} hasBin: true dependencies: - colorette: 2.0.19 + colorette: 2.0.20 defu: 6.1.2 https-proxy-agent: 5.0.1 mri: 1.2.0 - node-fetch-native: 1.0.1 - pathe: 1.1.0 + node-fetch-native: 1.4.0 + pathe: 1.1.1 tar: 6.1.13 transitivePeerDependencies: - supports-color @@ -10349,20 +12428,22 @@ packages: glob: 7.2.3 dev: true - /glob-promise@6.0.2(glob@8.1.0): - resolution: {integrity: sha512-Ni2aDyD1ekD6x8/+K4hDriRDbzzfuK4yKpqSymJ4P7IxbtARiOOuU+k40kbHM0sLIlbf1Qh0qdMkAHMZYE6XJQ==} - engines: {node: '>=16'} - peerDependencies: - glob: ^8.0.3 - dependencies: - '@types/glob': 8.1.0 - glob: 8.1.0 - dev: true - /glob-to-regexp@0.4.1: resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} dev: true + /glob@10.3.3: + resolution: {integrity: sha512-92vPiMb/iqpmEgsOoIDvTjc50wf9CCCvMzsi6W0JLPeUKE8TWP1a73PgqSrqy7iAZxaSD1YdzU7QZR5LF51MJw==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + dependencies: + foreground-child: 3.1.1 + jackspeak: 2.3.0 + minimatch: 9.0.3 + minipass: 7.0.3 + path-scurry: 1.10.1 + dev: true + /glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} dependencies: @@ -10374,17 +12455,6 @@ packages: path-is-absolute: 1.0.1 dev: true - /glob@8.1.0: - resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} - engines: {node: '>=12'} - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 5.1.6 - once: 1.4.0 - dev: true - /glob@9.3.1: resolution: {integrity: sha512-qERvJb7IGsnkx6YYmaaGvDpf77c951hICMdWaFXyH3PlVob8sbPJJyJX0kWkiCWyXUzoy9UOTNjGg0RbD8bYIw==} engines: {node: '>=16 || 14 >=14.17'} @@ -10509,8 +12579,8 @@ packages: through2: 2.0.5 dev: true - /handlebars@4.7.7: - resolution: {integrity: sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==} + /handlebars@4.7.8: + resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} engines: {node: '>=0.4.7'} hasBin: true dependencies: @@ -10563,10 +12633,6 @@ packages: has-symbols: 1.0.3 dev: true - /has-unicode@2.0.1: - resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} - dev: true - /has@1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} @@ -10639,6 +12705,11 @@ packages: engines: {node: '>=8'} dev: true + /html-tags@3.3.1: + resolution: {integrity: sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==} + engines: {node: '>=8'} + dev: true + /htmlparser2@8.0.1: resolution: {integrity: sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==} dependencies: @@ -10901,11 +12972,6 @@ packages: engines: {node: '>=12'} dev: false - /interpret@1.4.0: - resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} - engines: {node: '>= 0.10'} - dev: true - /intl-messageformat@10.2.5: resolution: {integrity: sha512-AievYMN6WLLHwBeCTv4aRKG+w3ZNyZtkObwgsKk3Q7GNTq8zDRvDbJSBQkb2OPeVCcAKcIXvak9FF/bRNavoww==} dependencies: @@ -10919,7 +12985,6 @@ packages: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} dependencies: loose-envify: 1.4.0 - dev: false /ip@1.1.8: resolution: {integrity: sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==} @@ -11098,6 +13163,11 @@ packages: is-path-inside: 3.0.3 dev: true + /is-interactive@1.0.0: + resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} + engines: {node: '>=8'} + dev: true + /is-map@2.0.2: resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} dev: true @@ -11278,15 +13348,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /isomorphic-unfetch@3.1.0: - resolution: {integrity: sha512-geDJjpoZ8N0kWexiwkX8F9NkTsXhetLPVbZFQ+JTW239QNOwvB0gniuR1Wc6f0AMTn7/mFGyXvHTifrCp/GH8Q==} - dependencies: - node-fetch: 2.6.9 - unfetch: 4.2.0 - transitivePeerDependencies: - - encoding - dev: true - /isstream@0.1.2: resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} dev: true @@ -11309,15 +13370,6 @@ packages: - supports-color dev: true - /istanbul-lib-report@3.0.0: - resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==} - engines: {node: '>=8'} - dependencies: - istanbul-lib-coverage: 3.2.0 - make-dir: 3.1.0 - supports-color: 7.2.0 - dev: true - /istanbul-lib-report@3.0.1: resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} engines: {node: '>=10'} @@ -11338,14 +13390,6 @@ packages: - supports-color dev: true - /istanbul-reports@3.1.5: - resolution: {integrity: sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==} - engines: {node: '>=8'} - dependencies: - html-escaper: 2.0.2 - istanbul-lib-report: 3.0.0 - dev: true - /istanbul-reports@3.1.6: resolution: {integrity: sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==} engines: {node: '>=8'} @@ -11354,8 +13398,17 @@ packages: istanbul-lib-report: 3.0.1 dev: true - /jake@10.8.5: - resolution: {integrity: sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==} + /jackspeak@2.3.0: + resolution: {integrity: sha512-uKmsITSsF4rUWQHzqaRUuyAir3fZfW3f202Ee34lz/gZCi970CPZwyQXLGNgWJvvZbvFyzeyGq0+4fcG/mBKZg==} + engines: {node: '>=14'} + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + dev: true + + /jake@10.8.7: + resolution: {integrity: sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==} engines: {node: '>=10'} hasBin: true dependencies: @@ -11556,7 +13609,7 @@ packages: micromatch: 4.0.5 walker: 1.0.8 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /jest-haste-map@29.6.2: @@ -11575,7 +13628,26 @@ packages: micromatch: 4.0.5 walker: 1.0.8 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 + dev: true + + /jest-haste-map@29.6.4: + resolution: {integrity: sha512-12Ad+VNTDHxKf7k+M65sviyynRoZYuL1/GTuhEVb8RYsNSNln71nANRb/faSyWvx0j+gHcivChXHIoMJrGYjog==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/graceful-fs': 4.1.6 + '@types/node': 18.16.3 + anymatch: 3.1.3 + fb-watchman: 2.0.2 + graceful-fs: 4.2.11 + jest-regex-util: 29.6.3 + jest-util: 29.6.3 + jest-worker: 29.6.4 + micromatch: 4.0.5 + walker: 1.0.8 + optionalDependencies: + fsevents: 2.3.3 dev: true /jest-leak-detector@29.6.2: @@ -11661,6 +13733,11 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dev: true + /jest-regex-util@29.6.3: + resolution: {integrity: sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dev: true + /jest-resolve-dependencies@29.6.2: resolution: {integrity: sha512-LGqjDWxg2fuQQm7ypDxduLu/m4+4Lb4gczc13v51VMZbVP5tSBILqVx8qfWcsdP8f0G7aIqByIALDB0R93yL+w==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11797,6 +13874,18 @@ packages: picomatch: 2.3.1 dev: true + /jest-util@29.6.3: + resolution: {integrity: sha512-QUjna/xSy4B32fzcKTSz1w7YYzgiHrjjJjevdRf61HYk998R5vVMMNmrHESYZVDS5DSWs+1srPLPKxXPkeSDOA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/node': 18.16.3 + chalk: 4.1.2 + ci-info: 3.8.0 + graceful-fs: 4.2.11 + picomatch: 2.3.1 + dev: true + /jest-validate@29.6.2: resolution: {integrity: sha512-vGz0yMN5fUFRRbpJDPwxMpgSXW1LDKROHfBopAvDcmD6s+B/s8WJrwi+4bfH4SdInBA5C3P3BI19dBtKzx1Arg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11843,6 +13932,16 @@ packages: supports-color: 8.1.1 dev: true + /jest-worker@29.6.4: + resolution: {integrity: sha512-6dpvFV4WjcWbDVGgHTWo/aupl8/LbBx2NSKfiwqf79xC/yeJjKHT1+StcKy/2KTmW16hE68ccKVOtXf+WZGz7Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@types/node': 18.16.3 + jest-util: 29.6.3 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: true + /jest@29.6.2(@types/node@18.16.3)(ts-node@10.9.1): resolution: {integrity: sha512-8eQg2mqFbaP7CwfsTpCxQ+sHzw1WuNWL5UUvjnWP4hx2riGz9fPSzYOaU5q8/GqWn1TfgZIVTqYJygbGbWAANg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11900,58 +13999,28 @@ packages: argparse: 2.0.1 /jsbn@0.1.1: - resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} - dev: true - - /jscodeshift@0.14.0(@babel/preset-env@7.20.2): - resolution: {integrity: sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==} - hasBin: true - peerDependencies: - '@babel/preset-env': ^7.1.6 - dependencies: - '@babel/core': 7.21.3 - '@babel/parser': 7.21.3 - '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.20.11(@babel/core@7.21.3) - '@babel/preset-env': 7.20.2(@babel/core@7.21.3) - '@babel/preset-flow': 7.18.6(@babel/core@7.21.3) - '@babel/preset-typescript': 7.21.0(@babel/core@7.21.3) - '@babel/register': 7.18.9(@babel/core@7.21.3) - babel-core: 7.0.0-bridge.0(@babel/core@7.21.3) - chalk: 4.1.2 - flow-parser: 0.198.2 - graceful-fs: 4.2.11 - micromatch: 4.0.5 - neo-async: 2.6.2 - node-dir: 0.1.17 - recast: 0.21.5 - temp: 0.8.4 - write-file-atomic: 2.4.3 - transitivePeerDependencies: - - supports-color + resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} dev: true - /jscodeshift@0.14.0(@babel/preset-env@7.21.4): + /jscodeshift@0.14.0(@babel/preset-env@7.22.10): resolution: {integrity: sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==} hasBin: true peerDependencies: '@babel/preset-env': ^7.1.6 dependencies: '@babel/core': 7.21.3 - '@babel/parser': 7.21.3 + '@babel/parser': 7.22.11 '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.20.11(@babel/core@7.21.3) - '@babel/preset-env': 7.21.4(@babel/core@7.21.3) - '@babel/preset-flow': 7.18.6(@babel/core@7.21.3) + '@babel/plugin-proposal-optional-chaining': 7.21.0(@babel/core@7.21.3) + '@babel/plugin-transform-modules-commonjs': 7.22.11(@babel/core@7.21.3) + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/preset-flow': 7.22.5(@babel/core@7.21.3) '@babel/preset-typescript': 7.21.0(@babel/core@7.21.3) - '@babel/register': 7.18.9(@babel/core@7.21.3) + '@babel/register': 7.22.5(@babel/core@7.21.3) babel-core: 7.0.0-bridge.0(@babel/core@7.21.3) chalk: 4.1.2 - flow-parser: 0.198.2 + flow-parser: 0.215.1 graceful-fs: 4.2.11 micromatch: 4.0.5 neo-async: 2.6.2 @@ -12059,12 +14128,6 @@ packages: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} dev: true - /json2mq@0.2.0: - resolution: {integrity: sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==} - dependencies: - string-convert: 0.2.1 - dev: false - /json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -12356,6 +14419,7 @@ packages: /lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + dev: true /lodash.get@4.4.2: resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} @@ -12448,6 +14512,11 @@ packages: dependencies: js-tokens: 4.0.0 + /lru-cache@10.0.1: + resolution: {integrity: sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==} + engines: {node: 14 || >=16.14} + dev: true + /lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} dependencies: @@ -12487,19 +14556,26 @@ packages: '@jridgewell/sourcemap-codec': 1.4.14 dev: true + /magic-string@0.30.3: + resolution: {integrity: sha512-B7xGbll2fG/VjP+SWg4sX3JynwIU0mjoTc6MPpKNuIvftk6u6vqhDnk1R80b8C2GBR6ywqy+1DcKBrevBg+bmw==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: true + /make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} dependencies: pify: 4.0.1 - semver: 5.7.1 + semver: 5.7.2 dev: true /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} dependencies: - semver: 6.3.0 + semver: 6.3.1 dev: true /make-dir@4.0.0: @@ -12541,14 +14617,13 @@ packages: resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==} dev: false - /markdown-to-jsx@7.1.8(react@18.2.0): - resolution: {integrity: sha512-rRSa1aFmFnpDRFAhv5vIkWM4nPaoB9vnzIjuIKa1wGupfn2hdCNeaQHKpu4/muoc8n8J7yowjTP2oncA4/Rbgg==} + /markdown-to-jsx@7.3.2(react@18.2.0): + resolution: {integrity: sha512-B+28F5ucp83aQm+OxNrPkS8z0tMKaeHiy0lHJs3LqCyDQFtWuenaIrkaVTgAm1pf1AU85LXltva86hlaT17i8Q==} engines: {node: '>= 10'} peerDependencies: react: '>= 0.14.0' dependencies: react: 18.2.0 - dev: true /matcher@1.1.1: resolution: {integrity: sha512-+BmqxWIubKTRKNWx/ahnCkk3mG8m7OturVlqq6HiojGJTd5hVYbgZm6WzcYPCoB+KBT4Vd6R7WSRG2OADNaCjg==} @@ -13109,6 +15184,13 @@ packages: brace-expansion: 2.0.1 dev: true + /minimatch@9.0.3: + resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + brace-expansion: 2.0.1 + dev: true + /minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -13141,6 +15223,11 @@ packages: engines: {node: '>=8'} dev: true + /minipass@7.0.3: + resolution: {integrity: sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==} + engines: {node: '>=16 || 14 >=14.17'} + dev: true + /minizlib@2.1.2: resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} engines: {node: '>= 8'} @@ -13289,8 +15376,8 @@ packages: http2-client: 1.3.5 dev: true - /node-fetch-native@1.0.1: - resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==} + /node-fetch-native@1.4.0: + resolution: {integrity: sha512-F5kfEj95kX8tkDhUCYdV8dg3/8Olx/94zB8+ZNthFs6Bz31UpUi8Xh40TN3thLwXgrwXry1pEg9lJ++tLWTcqA==} dev: true /node-fetch@2.6.7: @@ -13327,6 +15414,10 @@ packages: es6-promise: 3.3.1 dev: true + /node-releases@2.0.13: + resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==} + dev: true + /node-releases@2.0.8: resolution: {integrity: sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==} @@ -13376,15 +15467,6 @@ packages: path-key: 3.1.1 dev: true - /npmlog@5.0.1: - resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==} - dependencies: - are-we-there-yet: 2.0.0 - console-control-strings: 1.1.0 - gauge: 3.0.2 - set-blocking: 2.0.0 - dev: true - /nwsapi@2.2.2: resolution: {integrity: sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==} dev: true @@ -13535,16 +15617,17 @@ packages: format-util: 1.0.5 dev: true - /open@7.4.2: - resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} - engines: {node: '>=8'} + /open@8.4.0: + resolution: {integrity: sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==} + engines: {node: '>=12'} dependencies: + define-lazy-prop: 2.0.0 is-docker: 2.2.1 is-wsl: 2.2.0 dev: true - /open@8.4.0: - resolution: {integrity: sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==} + /open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} dependencies: define-lazy-prop: 2.0.0 @@ -13586,6 +15669,21 @@ packages: word-wrap: 1.2.3 dev: true + /ora@5.4.1: + resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} + engines: {node: '>=10'} + dependencies: + bl: 4.1.0 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-spinners: 2.9.0 + is-interactive: 1.0.0 + is-unicode-supported: 0.1.0 + log-symbols: 4.1.0 + strip-ansi: 6.0.1 + wcwidth: 1.0.1 + dev: true + /orval@6.15.0(openapi-types@12.1.3)(typescript@5.0.2): resolution: {integrity: sha512-SJx5PlwER6BzBMpnLkvFZkx50+rTvi2tvXgZyjMVjyxskgknb28BhFvYhMlKryT2UL0ihcQWHMnjgov4mScBwQ==} hasBin: true @@ -13797,6 +15895,14 @@ packages: /path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + /path-scurry@1.10.1: + resolution: {integrity: sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + lru-cache: 10.0.1 + minipass: 7.0.3 + dev: true + /path-scurry@1.6.1: resolution: {integrity: sha512-OW+5s+7cw6253Q4E+8qQ/u1fVvcJQCJo/VFD8pje+dbJCF1n5ZRMV2AEHbGp+5Q7jxQIYJxkHopnj6nzdGeZLA==} engines: {node: '>=14'} @@ -13813,8 +15919,8 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - /pathe@1.1.0: - resolution: {integrity: sha512-ODbEPR0KKHqECXW1GoxdDb+AZvULmXjVPy4rt+pGo2+TnjJTIPJQSVS6N63n8T2Ip+syHhbn52OewKicV0373w==} + /pathe@1.1.1: + resolution: {integrity: sha512-d+RQGp0MAYTIaDBIMmOfMwz3E+LOZnxx1HZd5R18mmCZY0QBlK0LDZfPc8FW8Ed2DlvsuE6PRjroDY+wg4+j/Q==} dev: true /pause-stream@0.0.11: @@ -13971,7 +16077,7 @@ packages: resolution: {integrity: sha512-Sz2Lkdxz6F2Pgnpi9U5Ng/WdWAUZxmHrNPoVlm3aAemxoy2Qy7LGjQg4uf8qKelDAUW94F4np3iH2YPf2qefcQ==} engines: {node: '>=10'} dependencies: - '@babel/runtime': 7.22.10 + '@babel/runtime': 7.22.11 dev: true /pony-cause@1.1.1: @@ -14377,6 +16483,13 @@ packages: side-channel: 1.0.4 dev: true + /qs@6.11.2: + resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.4 + dev: true + /query-string@6.14.1: resolution: {integrity: sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==} engines: {node: '>=6'} @@ -14410,6 +16523,10 @@ packages: resolution: {integrity: sha512-9QnLuG/kPVgWvMQ4aODhsBUFKOUmnbUnsSXACv+NCQZcHbeb+v8Lodp8OVxtRULN1/xOyYLLaL6npE6dMq5QTA==} dev: true + /ramda@0.29.0: + resolution: {integrity: sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==} + dev: true + /range-parser@1.2.1: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} @@ -14498,14 +16615,14 @@ packages: hasBin: true dependencies: '@babel/core': 7.21.3 - '@babel/generator': 7.21.3 + '@babel/generator': 7.22.10 ast-types: 0.14.2 commander: 2.20.3 doctrine: 3.0.0 estree-to-babel: 3.2.1 neo-async: 2.6.2 node-dir: 0.1.17 - resolve: 1.22.1 + resolve: 1.22.4 strip-indent: 3.0.0 transitivePeerDependencies: - supports-color @@ -14576,8 +16693,8 @@ packages: react: 18.2.0 dev: false - /react-inspector@6.0.1(react@18.2.0): - resolution: {integrity: sha512-cxKSeFTf7jpSSVddm66sKdolG90qURAX3g1roTeaN6x0YEbtWc8JpmFN9+yIqLNH2uEkYerWLtJZIXRIFuBKrg==} + /react-inspector@6.0.2(react@18.2.0): + resolution: {integrity: sha512-x+b7LxhmHXjHoU/VrFAzw5iutsILRoYyDq97EDYdFpPLcvqtEzk4ZSZSQjnFPbr5T57tLXnHcqFYoN1pI6u8uQ==} peerDependencies: react: ^16.8.4 || ^17.0.0 || ^18.0.0 dependencies: @@ -14753,6 +16870,41 @@ packages: engines: {node: '>=0.10.0'} dev: true + /react-remove-scroll-bar@2.3.4(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + react-style-singleton: 2.2.1(@types/react@18.2.20)(react@18.2.0) + tslib: 2.6.2 + dev: true + + /react-remove-scroll@2.5.5(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + react-remove-scroll-bar: 2.3.4(@types/react@18.2.20)(react@18.2.0) + react-style-singleton: 2.2.1(@types/react@18.2.20)(react@18.2.0) + tslib: 2.6.2 + use-callback-ref: 1.3.0(@types/react@18.2.20)(react@18.2.0) + use-sidecar: 1.1.2(@types/react@18.2.20)(react@18.2.0) + dev: true + /react-resize-detector@7.1.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-zXnPJ2m8+6oq9Nn8zsep/orts9vQv3elrpA+R8XTcW7DVVUJ9vwDwMXaBtykAYjMnkCIaOoK9vObyR7ZgFNlOw==} peerDependencies: @@ -14839,21 +16991,6 @@ packages: - '@types/react' dev: false - /react-slick@0.29.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-TGdOKE+ZkJHHeC4aaoH85m8RnFyWqdqRfAGkhd6dirmATXMZWAxOpTLmw2Ll/jPTQ3eEG7ercFr/sbzdeYCJXA==} - peerDependencies: - react: ^0.14.0 || ^15.0.1 || ^16.0.0 || ^17.0.0 || ^18.0.0 - react-dom: ^0.14.0 || ^15.0.1 || ^16.0.0 || ^17.0.0 || ^18.0.0 - dependencies: - classnames: 2.3.2 - enquire.js: 2.1.6 - json2mq: 0.2.0 - lodash.debounce: 4.0.8 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - resize-observer-polyfill: 1.5.1 - dev: false - /react-smooth@2.0.1(prop-types@15.8.1)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-Own9TA0GPPf3as4vSwFhDouVfXP15ie/wIHklhyKBH5AN6NFtdk0UpHBnonV11BtqDkAWlt40MOUc+5srmW7NA==} peerDependencies: @@ -14875,6 +17012,23 @@ packages: lodash: 4.17.21 dev: false + /react-style-singleton@2.2.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + get-nonce: 1.0.1 + invariant: 2.2.4 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /react-transition-group@2.9.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==} peerDependencies: @@ -15053,8 +17207,8 @@ packages: string_decoder: 0.10.31 dev: true - /readable-stream@2.3.7: - resolution: {integrity: sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==} + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} dependencies: core-util-is: 1.0.3 inherits: 2.0.4 @@ -15065,8 +17219,8 @@ packages: util-deprecate: 1.0.2 dev: true - /readable-stream@3.6.0: - resolution: {integrity: sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==} + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} dependencies: inherits: 2.0.4 @@ -15097,18 +17251,18 @@ packages: ast-types: 0.15.2 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.5.2 + tslib: 2.6.2 dev: true - /recast@0.23.1: - resolution: {integrity: sha512-RokaBcoxSjXUDzz1TXSZmZsSW6ZpLmlA3GGqJ8uuTrQ9hZhEz+4Tpsc+gRvYRJ2BU4H+ZyUlg91eSGDw7bwy7g==} + /recast@0.23.4: + resolution: {integrity: sha512-qtEDqIZGVcSZCHniWwZWbRy79Dc6Wp3kT/UmDA2RJKBPg7+7k51aQBZirHmUGn5uvHf2rg8DkjizrN26k61ATw==} engines: {node: '>= 4'} dependencies: assert: 2.0.0 ast-types: 0.16.1 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.5.2 + tslib: 2.6.2 dev: true /recharts-scale@0.4.5: @@ -15139,13 +17293,6 @@ packages: victory-vendor: 36.6.8 dev: false - /rechoir@0.6.2: - resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} - engines: {node: '>= 0.10'} - dependencies: - resolve: 1.22.1 - dev: true - /redent@3.0.0: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} @@ -15188,6 +17335,12 @@ packages: '@babel/runtime': 7.22.10 dev: true + /regenerator-transform@0.15.2: + resolution: {integrity: sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true + /regexp.prototype.flags@1.4.3: resolution: {integrity: sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==} engines: {node: '>= 0.4'} @@ -15218,6 +17371,18 @@ packages: unicode-match-property-value-ecmascript: 2.1.0 dev: true + /regexpu-core@5.3.2: + resolution: {integrity: sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==} + engines: {node: '>=4'} + dependencies: + '@babel/regjsgen': 0.8.0 + regenerate: 1.4.2 + regenerate-unicode-properties: 10.1.0 + regjsparser: 0.9.1 + unicode-match-property-ecmascript: 2.0.0 + unicode-match-property-value-ecmascript: 2.1.0 + dev: true + /regjsgen@0.7.1: resolution: {integrity: sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA==} dev: true @@ -15315,6 +17480,25 @@ packages: unist-util-visit: 2.0.3 dev: true + /remark-stringify@10.0.3: + resolution: {integrity: sha512-koyOzCMYoUHudypbj4XpnAKFbkddRMYZHwghnxd7ue5210WzGw6kOBwauJTRUMq16jsovXx8dYNvSSWP89kZ3A==} + dependencies: + '@types/mdast': 3.0.10 + mdast-util-to-markdown: 1.5.0 + unified: 10.1.2 + dev: false + + /remark@14.0.3: + resolution: {integrity: sha512-bfmJW1dmR2LvaMJuAnE88pZP9DktIFYXazkTfOIKZzi3Knk9lT0roItIA24ydOucI3bV/g/tXBA6hzqq3FV9Ew==} + dependencies: + '@types/mdast': 3.0.10 + remark-parse: 10.0.1 + remark-stringify: 10.0.3 + unified: 10.1.2 + transitivePeerDependencies: + - supports-color + dev: false + /remove-accents@0.4.2: resolution: {integrity: sha512-7pXIJqJOq5tFgG1A2Zxti3Ht8jJF337m4sowbuHsW30ZnkQFnDzy9qBNhgzX8ZLW4+UBcXiiR7SwR6pokHsxiA==} dev: false @@ -15446,7 +17630,7 @@ packages: engines: {node: '>=10.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /rollup@3.19.1: @@ -15454,7 +17638,15 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 + dev: true + + /rollup@3.28.1: + resolution: {integrity: sha512-R9OMQmIHJm9znrU3m3cpE8uhN0fGdXiawME7aZIpQqvpS/85+Vt1Hq1/yVIcYfOmaQiHjvXkQAoJukvLpau6Yw==} + engines: {node: '>=14.18.0', npm: '>=8.0.0'} + hasBin: true + optionalDependencies: + fsevents: 2.3.3 dev: true /rtl-css-js@1.16.1: @@ -15554,12 +17746,18 @@ packages: hasBin: true dev: true + /semver@5.7.2: + resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} + hasBin: true + requiresBuild: true + dev: true + /semver@6.3.0: resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} hasBin: true - /semver@7.0.0: - resolution: {integrity: sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==} + /semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true dev: true @@ -15622,10 +17820,6 @@ packages: - supports-color dev: true - /set-blocking@2.0.0: - resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - dev: true - /set-harmonic-interval@1.0.1: resolution: {integrity: sha512-AhICkFV84tBP1aWqPwLZqFvAwqEoVA9kxNMniGEUvzOlm4vLmOFLiTT3UZ6bziJTy4bOVpzWGTfSCbmaayGx8g==} engines: {node: '>=6.9'} @@ -15658,16 +17852,6 @@ packages: engines: {node: '>=8'} dev: true - /shelljs@0.8.5: - resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} - engines: {node: '>=4'} - hasBin: true - dependencies: - glob: 7.2.3 - interpret: 1.4.0 - rechoir: 0.6.2 - dev: true - /should-equal@2.0.0: resolution: {integrity: sha512-ZP36TMrK9euEuWQYBig9W55WPC7uo37qzAEmbjHz4gfyuXrEUgF8cUvQVO+w+d3OMfPvSRQJ22lSm8MQJ43LTA==} dependencies: @@ -15718,6 +17902,11 @@ packages: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} dev: true + /signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + dev: true + /simple-eval@1.0.0: resolution: {integrity: sha512-kpKJR+bqTscgC0xuAl2xHN6bB12lHjC2DCUfqjAx19bQyO3R2EVLOurm3H9AUltv/uFVcSCVNc6faegR+8NYLw==} engines: {node: '>=12'} @@ -15725,11 +17914,11 @@ packages: jsep: 1.3.8 dev: true - /simple-update-notifier@1.1.0: - resolution: {integrity: sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==} - engines: {node: '>=8.10.0'} + /simple-update-notifier@2.0.0: + resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} + engines: {node: '>=10'} dependencies: - semver: 7.0.0 + semver: 7.5.4 dev: true /sisteransi@1.0.5: @@ -16018,11 +18207,34 @@ packages: resolution: {integrity: sha512-siT1RiqlfQnGqgT/YzXVUNsom9S0H1OX+dpdGN1xkyYATo4I6sep5NmsRD/40s3IIOvlCq6akxkqG82urIZW1w==} dev: true - /storybook@7.0.2: - resolution: {integrity: sha512-/XBLhT9Vb14yNBcA9rlW15y+C6IsCA3kx5PKvK9kL10sKCi8invcY94UfCSisXe8HqsO3u6peumo2xpYucKMjw==} + /storybook-dark-mode@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-3V6XBhkUq63BF6KzyDBbfV5/8sYtF4UtVccH1tK+Lrd4p0tF8k7yHOvVDhFL9hexnKXcLEnbC+42YDTPvjpK+A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + dependencies: + '@storybook/addons': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.0.2 + '@storybook/global': 5.0.0 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + fast-deep-equal: 3.1.3 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /storybook@7.3.2: + resolution: {integrity: sha512-Vf1C5pfF5NHQsb+33NeBd3gLGhcwbT+v6WqqIdARV7LSByqKiWNgJl2ATgzm1b4ERJo8sHU+EiJZIovFWEElkg==} hasBin: true dependencies: - '@storybook/cli': 7.0.2 + '@storybook/cli': 7.3.2 transitivePeerDependencies: - bufferutil - encoding @@ -16055,10 +18267,6 @@ packages: engines: {node: '>=0.6.19'} dev: true - /string-convert@0.2.1: - resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} - dev: false - /string-length@4.0.2: resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} engines: {node: '>=10'} @@ -16162,6 +18370,13 @@ packages: ansi-regex: 6.0.1 dev: true + /strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: true + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -16454,8 +18669,8 @@ packages: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} dev: true - /synchronous-promise@2.0.16: - resolution: {integrity: sha512-qImOD23aDfnIDNqlG1NOehdB9IYsn1V9oByPjKY1nakv2MQYCEMyX033/q+aEtYCpmYK1cv2+NTmlH+ra6GA5A==} + /synchronous-promise@2.0.17: + resolution: {integrity: sha512-AsS729u2RHUfEra9xJrE39peJcc2stq2+poBXX8bcM08Y6g9j/i/PUzwNQqkaJde7Ntg1TO7bSREbR5sdosQ+g==} dev: true /synckit@0.8.5: @@ -16503,7 +18718,7 @@ packages: end-of-stream: 1.4.4 fs-constants: 1.0.0 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 dev: true /tar@6.1.13: @@ -16518,8 +18733,8 @@ packages: yallist: 4.0.0 dev: true - /telejson@7.0.4: - resolution: {integrity: sha512-J4QEuCnYGXAI9KSN7RXK0a0cOW2ONpjc4IQbInGZ6c3stvplLAYyZjTnScrRd8deXVjNCFV1wXcLC7SObDuQYA==} + /telejson@7.2.0: + resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==} dependencies: memoizerific: 1.11.3 dev: true @@ -16576,7 +18791,7 @@ packages: /through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} dependencies: - readable-stream: 2.3.7 + readable-stream: 2.3.8 xtend: 4.0.2 dev: true @@ -16620,6 +18835,10 @@ packages: dependencies: is-number: 7.0.0 + /tocbot@4.21.1: + resolution: {integrity: sha512-IfajhBTeg0HlMXu1f+VMbPef05QpDTsZ9X2Yn1+8npdaXsXg/+wrm9Ze1WG5OS1UDC3qJ5EQN/XOZ3gfXjPFCw==} + dev: true + /toggle-selection@1.0.6: resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} dev: false @@ -16772,6 +18991,10 @@ packages: /tslib@2.5.2: resolution: {integrity: sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA==} + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: true + /tsutils@3.21.0(typescript@5.0.2): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} @@ -16932,10 +19155,6 @@ packages: react-lifecycles-compat: 3.0.4 dev: false - /unfetch@4.2.0: - resolution: {integrity: sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==} - dev: true - /unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} @@ -17074,13 +19293,13 @@ packages: engines: {node: '>= 0.8'} dev: true - /unplugin@0.10.2: - resolution: {integrity: sha512-6rk7GUa4ICYjae5PrAllvcDeuT8pA9+j5J5EkxbMFaV+SalHhxZ7X2dohMzu6C3XzsMT+6jwR/+pwPNR3uK9MA==} + /unplugin@1.4.0: + resolution: {integrity: sha512-5x4eIEL6WgbzqGtF9UV8VEC/ehKptPXDS6L2b0mv4FRMkJxRtjaJfOWDd6a8+kYbqsjklix7yWP0N3SUepjXcg==} dependencies: - acorn: 8.8.2 + acorn: 8.10.0 chokidar: 3.5.3 webpack-sources: 3.2.3 - webpack-virtual-modules: 0.4.6 + webpack-virtual-modules: 0.5.0 dev: true /untildify@4.0.0: @@ -17098,6 +19317,17 @@ packages: escalade: 3.1.1 picocolors: 1.0.0 + /update-browserslist-db@1.0.11(browserslist@4.21.10): + resolution: {integrity: sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + dependencies: + browserslist: 4.21.10 + escalade: 3.1.1 + picocolors: 1.0.0 + dev: true + /uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: @@ -17122,6 +19352,21 @@ packages: querystring: 0.2.0 dev: false + /use-callback-ref@1.3.0(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /use-isomorphic-layout-effect@1.1.2(@types/react@18.2.20)(react@18.2.0): resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==} peerDependencies: @@ -17146,6 +19391,22 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true + /use-sidecar@1.1.2(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.9.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + detect-node-es: 1.1.0 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /use-sync-external-store@1.2.0(react@18.2.0): resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} peerDependencies: @@ -17182,10 +19443,6 @@ packages: engines: {node: '>= 0.4.0'} dev: true - /uuid-browser@3.1.0: - resolution: {integrity: sha512-dsNgbLaTrd6l3MMxTtouOCFw4CBFc/3a+GgYA2YyrJvyQ1u6q4pcu3ktLoUZ/VN/Aw9WsauazbgsgdfVWgAKQg==} - dev: true - /uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -17193,7 +19450,6 @@ packages: /uuid@9.0.0: resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} hasBin: true - dev: false /uvu@0.5.6: resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} @@ -17407,7 +19663,7 @@ packages: rollup: 3.19.1 sass: 1.59.3 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /vm2@3.9.19: @@ -17524,8 +19780,8 @@ packages: engines: {node: '>=10.13.0'} dev: true - /webpack-virtual-modules@0.4.6: - resolution: {integrity: sha512-5tyDlKLqPfMqjT3Q9TAqf2YqjwmnUleZwzJi1A5qXnlBCdj2AtOJ6wAWdglTIDOPgOiOrXeBeFcsQ8+aGQ6QbA==} + /webpack-virtual-modules@0.5.0: + resolution: {integrity: sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw==} dev: true /websocket-driver@0.7.4: @@ -17618,19 +19874,6 @@ packages: isexe: 2.0.0 dev: true - /wide-align@1.1.5: - resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} - dependencies: - string-width: 4.2.3 - dev: true - - /widest-line@3.1.0: - resolution: {integrity: sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==} - engines: {node: '>=8'} - dependencies: - string-width: 4.2.3 - dev: true - /word-wrap@1.2.3: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} engines: {node: '>=0.10.0'} @@ -17657,6 +19900,15 @@ packages: string-width: 4.2.3 strip-ansi: 6.0.1 + /wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + dev: true + /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true @@ -17704,6 +19956,19 @@ packages: optional: true dev: true + /ws@8.13.0: + resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: true + /xml-name-validator@4.0.0: resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==} engines: {node: '>=12'} diff --git a/airbyte-webapp/scripts/license-check.js b/airbyte-webapp/scripts/license-check.js index 5def59b8dd5..cee692b406a 100644 --- a/airbyte-webapp/scripts/license-check.js +++ b/airbyte-webapp/scripts/license-check.js @@ -3,8 +3,6 @@ const { promisify } = require("util"); const checker = require("license-checker"); -const { version } = require("../package.json"); - /** * A list of all the allowed licenses that production dependencies can have. */ @@ -52,7 +50,7 @@ const ALLOWED_DEV_LICENSES = [...ALLOWED_LICENSES, "ODC-By-1.0", "MPL-2.0"]; /** * A list of all packages that should be excluded from license checking. */ -const IGNORED_PACKAGES = [`airbyte-webapp@${version}`]; +const IGNORED_PACKAGES = []; /** * Overwrite licenses for specific packages manually, e.g. because they can't be detected properly. @@ -71,6 +69,10 @@ const params = { function validateLicenes(licenses, allowedLicenes, usedOverwrites) { let licensesValid = true; for (const [pkg, info] of Object.entries(licenses)) { + if (pkg.startsWith("airbyte-webapp@")) { + // Skip our project itself + continue; + } let license = Array.isArray(info.licenses) ? `(${info.licenses.join(" OR ")})` : info.licenses; if (LICENSE_OVERWRITES[pkg]) { license = LICENSE_OVERWRITES[pkg]; diff --git a/airbyte-webapp/scripts/validate-lock-files.js b/airbyte-webapp/scripts/validate-lock-files.js new file mode 100644 index 00000000000..756f9cb7358 --- /dev/null +++ b/airbyte-webapp/scripts/validate-lock-files.js @@ -0,0 +1,25 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +function assertFileExists(filepath) { + if (!fs.existsSync(filepath)) { + throw new Error(`File ${filepath} does not exist`); + } +} + +function assertFileNotExists(filepath) { + if (fs.existsSync(filepath)) { + throw new Error(`File ${filepath} exists but should not`); + } +} + +try { + assertFileExists(path.join(__dirname, "..", "pnpm-lock.yaml")); + assertFileNotExists(path.join(__dirname, "..", "package-lock.json")); + assertFileNotExists(path.join(__dirname, "..", "yarn.lock")); + + console.log("Lock file validation successful."); +} catch (error) { + console.error(`Lock file validation failed: ${error.message}`); + process.exit(1); +} diff --git a/airbyte-webapp/src/App.tsx b/airbyte-webapp/src/App.tsx index 9725f174d4f..d455f0185bd 100644 --- a/airbyte-webapp/src/App.tsx +++ b/airbyte-webapp/src/App.tsx @@ -8,6 +8,7 @@ import { ApiErrorBoundary } from "components/common/ApiErrorBoundary"; import { config } from "config"; import { QueryProvider, useGetInstanceConfiguration } from "core/api"; import { AnalyticsProvider } from "core/services/analytics"; +import { OSSAuthService } from "core/services/auth"; import { defaultOssFeatures, FeatureService } from "core/services/features"; import { I18nProvider } from "core/services/i18n"; import { BlockerService } from "core/services/navigation"; @@ -19,7 +20,6 @@ import { ModalServiceProvider } from "hooks/services/Modal"; import { NotificationService } from "hooks/services/Notification"; import { AirbyteThemeProvider } from "hooks/theme/useAirbyteTheme"; import { ConnectorBuilderTestInputProvider } from "services/connectorBuilder/ConnectorBuilderTestInputService"; -import { KeycloakAuthService } from "services/KeycloakAuthService"; import LoadingPage from "./components/LoadingPage"; import { ConfigServiceProvider } from "./config"; @@ -33,7 +33,7 @@ const StyleProvider: React.FC> = ({ children }) const Services: React.FC> = ({ children }) => ( - + @@ -45,7 +45,7 @@ const Services: React.FC> = ({ children }) => ( - + ); diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss new file mode 100644 index 00000000000..979694d3639 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss @@ -0,0 +1,15 @@ +@use "scss/colors"; + +.tooltipWrapper { + background-color: colors.$foreground !important; + color: colors.$dark-blue span { + color: colors.$dark-blue; + } +} + +:export { + graphColor: colors.$grey-100; + chartHoverFill: colors.$grey-50; + tooltipLabelColor: colors.$dark-blue; + tooltipItemColor: colors.$grey-700; +} diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx new file mode 100644 index 00000000000..6034f4852d7 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx @@ -0,0 +1,48 @@ +import byteSize from "byte-size"; +import { Bar, BarChart, ResponsiveContainer, Tooltip, XAxis } from "recharts"; + +import { Box } from "components/ui/Box"; + +import styles from "./DataMovedGraph.module.scss"; + +const sampleData: Array<{ date: number; bytes: number }> = []; +for (let i = 1; i <= 30; i++) { + sampleData.push({ date: Date.UTC(2023, 7, i), bytes: Math.round(Math.random() * 1000 + 200) }); +} + +export const DataMovedGraph: React.FC = () => { + return ( + + + new Date(x).toLocaleDateString(undefined, { month: "short", day: "numeric" })} + /> + {new Date(value).toLocaleDateString()}} + formatter={(value: number) => { + // The type cast is unfortunately necessary, due to broken typing in recharts. + // What we return is a [string, undefined], and the library accepts this as well, but the types + // require the first element to be of the same type as value, which isn't what the formatter + // is supposed to do: https://github.com/recharts/recharts/issues/3008 + const prettyvalues = byteSize(value); + return [ + <> + {prettyvalues.value} {prettyvalues.long} + , + undefined, + ] as unknown as [number, string]; + }} + /> + + + + ); +}; diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts b/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts new file mode 100644 index 00000000000..4dae61d273e --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts @@ -0,0 +1 @@ +export * from "./DataMovedGraph"; diff --git a/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx b/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx new file mode 100644 index 00000000000..c89b9f26701 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx @@ -0,0 +1,35 @@ +import { useState } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { Tabs } from "components/ui/Tabs"; +import { ButtonTab } from "components/ui/Tabs/ButtonTab"; + +import { DataMovedGraph } from "../data-moved-graph"; + +export const HistoricalOverview: React.FC = () => { + const [selectedTab, setSelectedTab] = useState<"uptimeStatus" | "dataMoved">("uptimeStatus"); + + return ( + + + } + isActive={selectedTab === "uptimeStatus"} + onSelect={() => setSelectedTab("uptimeStatus")} + /> + } + isActive={selectedTab === "dataMoved"} + onSelect={() => setSelectedTab("dataMoved")} + /> + + + {selectedTab === "uptimeStatus" &&
uptime
} + {selectedTab === "dataMoved" && } + + + ); +}; diff --git a/airbyte-webapp/src/area/connection/components/historical-overview/index.ts b/airbyte-webapp/src/area/connection/components/historical-overview/index.ts new file mode 100644 index 00000000000..77dbeb0ed2a --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/historical-overview/index.ts @@ -0,0 +1 @@ +export * from "./HistoricalOverview"; diff --git a/airbyte-webapp/src/area/connection/components/index.ts b/airbyte-webapp/src/area/connection/components/index.ts new file mode 100644 index 00000000000..c2536b3fb4c --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/index.ts @@ -0,0 +1,2 @@ +export * from "./data-moved-graph"; +export * from "./historical-overview"; diff --git a/airbyte-webapp/src/area/connection/types/index.ts b/airbyte-webapp/src/area/connection/types/index.ts new file mode 100644 index 00000000000..1e212050ff1 --- /dev/null +++ b/airbyte-webapp/src/area/connection/types/index.ts @@ -0,0 +1 @@ +export * from "./normalization"; diff --git a/airbyte-webapp/src/area/connection/types/normalization.ts b/airbyte-webapp/src/area/connection/types/normalization.ts new file mode 100644 index 00000000000..0f927ff5954 --- /dev/null +++ b/airbyte-webapp/src/area/connection/types/normalization.ts @@ -0,0 +1,4 @@ +export enum NormalizationType { + basic = "basic", + raw = "raw", +} diff --git a/airbyte-webapp/src/area/connection/utils/index.ts b/airbyte-webapp/src/area/connection/utils/index.ts index 4883ec258f0..9dfeef757e9 100644 --- a/airbyte-webapp/src/area/connection/utils/index.ts +++ b/airbyte-webapp/src/area/connection/utils/index.ts @@ -1,4 +1,5 @@ export * from "./computeStreamStatus"; export * from "./dataTypes"; +export * from "./operation"; export * from "./useStreamsStatuses"; export * from "./validateCronExpression"; diff --git a/airbyte-webapp/src/core/domain/connection/operation.ts b/airbyte-webapp/src/area/connection/utils/operation.ts similarity index 87% rename from airbyte-webapp/src/core/domain/connection/operation.ts rename to airbyte-webapp/src/area/connection/utils/operation.ts index 8397930f1ec..0563ceb3f6e 100644 --- a/airbyte-webapp/src/core/domain/connection/operation.ts +++ b/airbyte-webapp/src/area/connection/utils/operation.ts @@ -1,11 +1,6 @@ import { DbtOperationRead } from "components/connection/TransformationHookForm"; -import { OperationCreate, OperationRead, OperatorType } from "core/request/AirbyteClient"; - -export enum NormalizationType { - basic = "basic", - raw = "raw", -} +import { OperationCreate, OperationRead, OperatorType } from "core/api/types/AirbyteClient"; export const isDbtTransformation = (op: OperationRead): op is DbtOperationRead => { return op.operatorConfiguration.operatorType === OperatorType.dbt; diff --git a/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts b/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts index 99529c76e90..9e7db41cd1a 100644 --- a/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts +++ b/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts @@ -8,11 +8,10 @@ import { useLateMultiplierExperiment, } from "components/connection/StreamStatus/streamStatusUtils"; -import { useListStreamsStatuses } from "core/api"; -import { StreamStatusRead } from "core/request/AirbyteClient"; +import { useListStreamsStatuses, useGetConnection } from "core/api"; +import { StreamStatusRead } from "core/api/types/AirbyteClient"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; import { useExperiment } from "hooks/services/Experiment"; -import { useGetConnection } from "hooks/services/useConnectionHook"; import { AirbyteStreamAndConfigurationWithEnforcedStream, diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.module.scss b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.module.scss new file mode 100644 index 00000000000..cce50d9c4bb --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.module.scss @@ -0,0 +1,21 @@ +@use "scss/colors"; +@use "scss/variables"; + +.suggestedConnectors { + position: relative; + background: colors.$blue-50; + padding: variables.$spacing-xl; + border-radius: variables.$border-radius-md; + + &__dismiss { + position: absolute; + top: variables.$spacing-md; + right: variables.$spacing-md; + } + + &__grid { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: variables.$spacing-xl; + } +} diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx new file mode 100644 index 00000000000..b88e443d518 --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx @@ -0,0 +1,97 @@ +import React from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { ConnectorButton } from "components/source/SelectConnector/ConnectorButton"; +import { Box } from "components/ui/Box"; +import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; +import { Icon } from "components/ui/Icon"; +import { Text } from "components/ui/Text"; + +import { ConnectorDefinition } from "core/domain/connector"; +import { isSourceDefinition } from "core/domain/connector/source"; +import { useLocalStorage } from "core/utils/useLocalStorage"; +import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; +import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; + +import styles from "./SuggestedConnectors.module.scss"; + +interface SuggestedConnectorsProps { + definitionIds: string[]; + onConnectorButtonClick: (definition: ConnectorDefinition) => void; +} + +export const SuggestedConnectorsUnmemoized: React.FC = ({ + definitionIds, + onConnectorButtonClick, +}) => { + const { formatMessage } = useIntl(); + const { sourceDefinitionMap } = useSourceDefinitionList(); + const { destinationDefinitionMap } = useDestinationDefinitionList(); + const [showSuggestedConnectors, setShowSuggestedConnectors] = useLocalStorage( + "airbyte_connector-grid-show-suggested-connectors", + true + ); + + const definitions = definitionIds + .map((definitionId) => sourceDefinitionMap.get(definitionId) ?? destinationDefinitionMap.get(definitionId)) + .filter((definitionId): definitionId is ConnectorDefinition => Boolean(definitionId)) + // We want to display at most 3 suggested connectors + .slice(0, 3); + + // If no valid suggested connectors are provided, don't render anything + if (definitions.length === 0) { + return null; + } + + if (!showSuggestedConnectors) { + return ( + + + + + + + + ); + } + + // It's a good enough proxy to check if the first definition is a source or destination + const titleKey = isSourceDefinition(definitions[0]) + ? "sources.suggestedSources" + : "destinations.suggestedDestinations"; + + return ( +
+ +
+ + + + +
+ +
+ {definitions.map((definition) => ( + onConnectorButtonClick(definition)} + key={isSourceDefinition(definition) ? definition.sourceDefinitionId : definition.destinationDefinitionId} + /> + ))} +
+
+ ); +}; + +export const SuggestedConnectors = React.memo(SuggestedConnectorsUnmemoized); diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts new file mode 100644 index 00000000000..0fb6a034d5b --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts @@ -0,0 +1 @@ +export * from "./SuggestedConnectors"; diff --git a/airbyte-webapp/src/area/connector/components/index.ts b/airbyte-webapp/src/area/connector/components/index.ts new file mode 100644 index 00000000000..0fb6a034d5b --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/index.ts @@ -0,0 +1 @@ +export * from "./SuggestedConnectors"; diff --git a/airbyte-webapp/src/area/connector/types/connectionConfiguration.ts b/airbyte-webapp/src/area/connector/types/connectionConfiguration.ts new file mode 100644 index 00000000000..ffe3815423f --- /dev/null +++ b/airbyte-webapp/src/area/connector/types/connectionConfiguration.ts @@ -0,0 +1 @@ +export type ConnectionConfiguration = unknown; diff --git a/airbyte-webapp/src/area/connector/types/index.ts b/airbyte-webapp/src/area/connector/types/index.ts new file mode 100644 index 00000000000..ae24f1ae88c --- /dev/null +++ b/airbyte-webapp/src/area/connector/types/index.ts @@ -0,0 +1 @@ +export * from "./connectionConfiguration"; diff --git a/airbyte-webapp/src/area/connector/utils/index.ts b/airbyte-webapp/src/area/connector/utils/index.ts index 7f6a03c157a..3515cab93f4 100644 --- a/airbyte-webapp/src/area/connector/utils/index.ts +++ b/airbyte-webapp/src/area/connector/utils/index.ts @@ -1,2 +1,4 @@ export { ConnectorIds } from "./constants"; export { SvgIcon } from "./SvgIcon"; +export * from "./useSuggestedDestinations"; +export * from "./useSuggestedSources"; diff --git a/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts b/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts new file mode 100644 index 00000000000..9824a516184 --- /dev/null +++ b/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts @@ -0,0 +1,11 @@ +import { useMemo } from "react"; + +import { useExperiment } from "hooks/services/Experiment/ExperimentService"; + +export const useSuggestedDestinations = () => { + const suggestedDestinationConnectors = useExperiment("connector.suggestedDestinationConnectors", ""); + return useMemo( + () => suggestedDestinationConnectors.split(",").map((id) => id.trim()), + [suggestedDestinationConnectors] + ); +}; diff --git a/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts b/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts new file mode 100644 index 00000000000..dfd65e33836 --- /dev/null +++ b/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts @@ -0,0 +1,8 @@ +import { useMemo } from "react"; + +import { useExperiment } from "hooks/services/Experiment/ExperimentService"; + +export const useSuggestedSources = () => { + const suggestedSourceConnectors = useExperiment("connector.suggestedSourceConnectors", ""); + return useMemo(() => suggestedSourceConnectors.split(",").map((id) => id.trim()), [suggestedSourceConnectors]); +}; diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss deleted file mode 100644 index bcad38ce6b0..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss +++ /dev/null @@ -1,55 +0,0 @@ -@use "../../scss/colors"; -@use "../../scss/variables"; - -.container { - display: flex; - padding: variables.$spacing-md; - width: 220px; - align-items: center; -} - -.details { - width: 160px; - margin-left: variables.$spacing-md; - display: flex; - flex-direction: column; - font-weight: normal; -} - -.entityIcon { - height: 30px; - width: 30px; -} - -.connectionName { - font-size: variables.$font-size-lg; - color: colors.$dark-blue-900; - text-align: left; - margin-right: variables.$spacing-md; -} - -.connectorDetails { - display: flex; - justify-content: flex-start; - align-items: center; -} - -.connectorName { - font-size: 11px; - margin-top: 1px; - color: colors.$grey-300; - text-align: left; - word-wrap: break-word; -} - -.fullWidth { - width: 100%; - - .details { - width: 100%; - - .connectorDetails { - justify-content: space-between; - } - } -} diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx deleted file mode 100644 index 49cc04ad683..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import classnames from "classnames"; -import React from "react"; - -import { ReleaseStageBadge } from "components/ReleaseStageBadge"; - -import { SvgIcon } from "area/connector/utils"; -import { ReleaseStage } from "core/request/AirbyteClient"; - -import styles from "./ConnectorCard.module.scss"; - -export interface ConnectorCardProps { - connectionName: string; - icon?: string; - connectorName?: string; - releaseStage?: ReleaseStage; - fullWidth?: boolean; -} - -export const ConnectorCard: React.FC = ({ - connectionName, - connectorName, - icon, - releaseStage, - fullWidth, -}) => ( -
- {icon && ( -
- -
- )} -
-
-
{connectionName}
- {releaseStage && } -
- {connectorName &&
{connectorName}
} -
-
-); diff --git a/airbyte-webapp/src/components/ConnectorCard/index.tsx b/airbyte-webapp/src/components/ConnectorCard/index.tsx deleted file mode 100644 index 77f1bb6271b..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./ConnectorCard"; diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx index e865363ea6e..4cc164de370 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx @@ -4,7 +4,7 @@ import { TestWrapper, TestSuspenseBoundary, mockConnection } from "test-utils"; import { StatusCell } from "./StatusCell"; -jest.mock("hooks/services/useConnectionHook", () => ({ +jest.mock("core/api", () => ({ useConnectionList: jest.fn(() => ({ connections: [], })), @@ -18,7 +18,7 @@ jest.mock("hooks/services/useConnectionHook", () => ({ const mockId = "mock-id"; -jest.doMock("hooks/services/useConnectionHook", () => ({ +jest.doMock("core/api", () => ({ useEnableConnection: () => ({ mutateAsync: jest.fn(), isLoading: false, diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx index fbda0dc04f0..df64c1b8b1e 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx @@ -4,8 +4,8 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Switch } from "components/ui/Switch"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; -import { useEnableConnection, useSyncConnection } from "hooks/services/useConnectionHook"; +import { useEnableConnection, useSyncConnection } from "core/api"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import styles from "./StatusCellControl.module.scss"; diff --git a/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx index af301059360..ceb97c6ebd9 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx @@ -67,13 +67,16 @@ const StreamsPerStatus: React.FC<{ enabledStreams: AirbyteStreamAndConfigurationWithEnforcedStream[]; }> = ({ streamStatuses, enabledStreams }) => { const sortedStreamsMap = sortStreams(enabledStreams, streamStatuses); - const filteredAndSortedStreams = Object.entries(sortedStreamsMap).filter(([, streams]) => !!streams.length); + const filteredAndSortedStreams = Object.entries(sortedStreamsMap).filter(([, streams]) => !!streams.length) as Array< + [ConnectionStatusIndicatorStatus, StreamWithStatus[]] + >; + return ( <> {filteredAndSortedStreams.map(([statusType, streams]) => (
- + {streams.length}
diff --git a/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss b/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss index 8714e91c1a1..a52d468db2c 100644 --- a/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss +++ b/airbyte-webapp/src/components/GroupControls/GroupControls.module.scss @@ -58,7 +58,7 @@ $border-width: variables.$border-thick; // only apply padding when there are children, so that empty group sections border is just a single line > :first-child { - padding-top: calc($group-spacing + $title-height/2); + padding-top: calc($group-spacing + variables.$spacing-md); } > div { @@ -68,4 +68,9 @@ $border-width: variables.$border-thick; > :last-child { margin-bottom: 0; } + + &--error { + border-color: colors.$red-100; + box-shadow: 0 $border-width colors.$red-100 inset; + } } diff --git a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx index c69eb8bcbf6..68ddbf9ff40 100644 --- a/airbyte-webapp/src/components/GroupControls/GroupControls.tsx +++ b/airbyte-webapp/src/components/GroupControls/GroupControls.tsx @@ -1,5 +1,8 @@ import classNames from "classnames"; import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { PropertyError } from "views/Connector/ConnectorForm/components/Property/PropertyError"; import styles from "./GroupControls.module.scss"; @@ -8,6 +11,7 @@ interface GroupControlsProps { control?: React.ReactNode; controlClassName?: string; name?: string; + error?: string; } const GroupControls: React.FC> = ({ @@ -16,6 +20,7 @@ const GroupControls: React.FC> = ({ children, name, controlClassName, + error, }) => { return ( // This outer div is necessary for .content > :first-child padding to be properly applied in the case of nested GroupControls @@ -25,10 +30,15 @@ const GroupControls: React.FC> = ({
{label}
{control}
-
+
{children}
+ {error && ( + + + + )} ); }; diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx b/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx index 11f594931d0..a507be310e6 100644 --- a/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx +++ b/airbyte-webapp/src/components/NewJobItem/JobLogsModalContent.tsx @@ -3,6 +3,7 @@ import { useIntl } from "react-intl"; import { useDebounce } from "react-use"; import { AttemptDetails } from "components/JobItem/components/AttemptDetails"; +import { LinkToAttemptButton } from "components/JobItem/components/LinkToAttemptButton"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; import { ListBox } from "components/ui/ListBox"; @@ -19,16 +20,17 @@ import { VirtualLogs } from "./VirtualLogs"; interface JobLogsModalContentProps { jobId: number; + initialAttemptIndex?: number; } -export const JobLogsModalContent: React.FC = ({ jobId }) => { +export const JobLogsModalContent: React.FC = ({ jobId, initialAttemptIndex }) => { const searchInputRef = useRef(null); const [inputValue, setInputValue] = useState(""); const job = useJobInfoWithoutLogs(jobId); const [highlightedMatchIndex, setHighlightedMatchIndex] = useState(undefined); const [matchingLines, setMatchingLines] = useState([]); const highlightedMatchingLineNumber = highlightedMatchIndex !== undefined ? highlightedMatchIndex + 1 : undefined; - const [selectedAttemptIndex, setSelectedAttemptIndex] = useState(job.attempts.length - 1); + const [selectedAttemptIndex, setSelectedAttemptIndex] = useState(initialAttemptIndex ?? job.attempts.length - 1); const jobAttempt = useAttemptForJob(jobId, selectedAttemptIndex); const logLines = useCleanLogs(jobAttempt); const firstMatchIndex = 0; @@ -146,9 +148,10 @@ export const JobLogsModalContent: React.FC = ({ jobId /> -
+ + -
+ diff --git a/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx b/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx index e483b8f8c3b..5dd8dbd1649 100644 --- a/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx +++ b/airbyte-webapp/src/components/NewJobItem/JobLogsModalFailureMessage.tsx @@ -39,7 +39,7 @@ export const JobLogsModalFailureMessage: React.FC + {failureToShow === "internal" && ( {({ open }) => ( diff --git a/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx b/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx index 7f762f92032..8956550f670 100644 --- a/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx +++ b/airbyte-webapp/src/components/NewJobItem/NewJobItem.tsx @@ -1,7 +1,7 @@ import { faEllipsisV } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; -import { Suspense, useRef } from "react"; +import { Suspense, useCallback, useRef } from "react"; import { FormattedDate, FormattedMessage, FormattedTimeParts, useIntl } from "react-intl"; import { useEffectOnce } from "react-use"; @@ -60,9 +60,31 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { useEffectOnce(() => { if (attemptLink.jobId === String(jobWithAttempts.job.id)) { wrapperRef.current?.scrollIntoView(); + openJobLogsModal(attemptLink.attemptId ? Number(attemptLink.attemptId) - 1 : undefined); } }); + const openJobLogsModal = useCallback( + (initialAttemptIndex?: number) => { + openModal({ + size: "full", + title: formatMessage({ id: "jobHistory.logs.title" }, { connectionName: connection.name }), + content: () => ( + + + + } + > + + + ), + }); + }, + [connection.name, formatMessage, jobWithAttempts.job.id, openModal] + ); + const handleClick = (optionClicked: DropdownMenuOptionType) => { switch (optionClicked.value) { case ContextMenuOptions.DownloadLogs: @@ -120,21 +142,7 @@ export const NewJobItem: React.FC = ({ jobWithAttempts }) => { }); break; case ContextMenuOptions.OpenLogsModal: - openModal({ - size: "full", - title: formatMessage({ id: "jobHistory.logs.title" }, { connectionName: connection.name }), - content: () => ( - - - - } - > - - - ), - }); + openJobLogsModal(); break; case ContextMenuOptions.CopyLinkToJob: const url = new URL(window.location.href); diff --git a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx b/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx index deebdd0cc75..ccce9fc7ba2 100644 --- a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx +++ b/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx @@ -62,7 +62,7 @@ const VirtualLogsUnmemoized: React.FC = ({ logLines, searchTer if (listRef.current === null || !logLines.length) { return; } - listRef?.current.scrollToItem(logLines.length); + listRef?.current.scrollToItem(logLines.length, "end"); }, [logLines.length]); useEffect(() => { diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss new file mode 100644 index 00000000000..38093bc6a93 --- /dev/null +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss @@ -0,0 +1,3 @@ +.notificationItemField { + grid-column: 1 / 1; +} diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx index 2a0796a3868..dabf90db27c 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx @@ -8,17 +8,24 @@ import { Tooltip } from "components/ui/Tooltip"; import { FeatureItem, useFeature } from "core/services/features"; +import styles from "./NotificationItemField.module.scss"; import { NotificationSettingsFormValues } from "./NotificationSettingsForm"; import { SlackNotificationUrlInput } from "./SlackNotificationUrlInput"; import { TestWebhookButton } from "./TestWebhookButton"; interface NotificationItemFieldProps { emailNotificationRequired?: boolean; + slackNotificationUnsupported?: boolean; name: keyof NotificationSettingsFormValues; } -export const NotificationItemField: React.FC = ({ emailNotificationRequired, name }) => { +export const NotificationItemField: React.FC = ({ + emailNotificationRequired, + slackNotificationUnsupported, + name, +}) => { const emailNotificationsFeature = useFeature(FeatureItem.EmailNotifications); + const atLeastOneNotificationTypeEnableable = emailNotificationsFeature || !slackNotificationUnsupported; const { setValue, trigger } = useFormContext(); const field = useWatch({ name }); @@ -36,9 +43,13 @@ export const NotificationItemField: React.FC = ({ em } }; + if (!atLeastOneNotificationTypeEnableable) { + return null; + } + return ( <> -
+
@@ -48,6 +59,7 @@ export const NotificationItemField: React.FC = ({ em
+ {emailNotificationsFeature && ( {!emailNotificationRequired && ( @@ -60,11 +72,20 @@ export const NotificationItemField: React.FC = ({ em )} )} - - - - - + + {!slackNotificationUnsupported && ( + <> + + + + + + + )} ); }; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx index f6220c636f8..63916351edb 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx @@ -27,6 +27,7 @@ import { formValuesToNotificationSettings } from "./formValuesToNotificationSett import { NotificationItemField } from "./NotificationItemField"; import styles from "./NotificationSettingsForm.module.scss"; import { notificationSettingsToFormValues } from "./notificationSettingsToFormValues"; +import { useExperiment } from "../../hooks/services/Experiment"; interface NotificationSettingsFormProps { updateNotificationSettings: (notificationSettings: NotificationSettings) => Promise; @@ -34,6 +35,7 @@ interface NotificationSettingsFormProps { export const NotificationSettingsForm: React.FC = ({ updateNotificationSettings }) => { const emailNotificationsFeatureEnabled = useFeature(FeatureItem.EmailNotifications); + const breakingChangeNotificationsExperimentEnabled = useExperiment("settings.breakingChangeNotifications", false); const { notificationSettings, email } = useCurrentWorkspace(); const defaultValues = notificationSettingsToFormValues(notificationSettings); const testWebhook = useTryNotificationWebhook(); @@ -54,7 +56,7 @@ export const NotificationSettingsForm: React.FC = notificationKeys.map(async (key) => { const notification = values[key]; - // If slack is not set as a notification type, or if the webhook has not changed, we can skip the validation + // If Slack is not set as a notification type, or if the webhook has not changed, we can skip the validation if ( !notification.slack || (!methods.formState.dirtyFields[key]?.slack && !methods.formState.dirtyFields[key]?.slackWebhookLink) @@ -109,7 +111,7 @@ export const NotificationSettingsForm: React.FC = trackError(e, { name: "notification_settings_update_error", formValues: values, - requestPayloas: newNotificationSettings, + requestPayload: newNotificationSettings, }); registerNotification({ id: "notification_settings_update", @@ -162,8 +164,18 @@ export const NotificationSettingsForm: React.FC = - + + {breakingChangeNotificationsExperimentEnabled && ( + <> + + + + )}
@@ -188,6 +200,8 @@ export interface NotificationSettingsFormValues { sendOnConnectionUpdateActionRequired: NotificationItemFieldValue; sendOnSyncDisabled: NotificationItemFieldValue; sendOnSyncDisabledWarning: NotificationItemFieldValue; + sendOnBreakingChangeWarning: NotificationItemFieldValue; + sendOnBreakingChangeSyncsDisabled: NotificationItemFieldValue; } const notificationItemSchema: SchemaOf = yup.object({ @@ -206,6 +220,8 @@ const validationSchema: SchemaOf = yup.object({ sendOnConnectionUpdateActionRequired: notificationItemSchema, sendOnSyncDisabled: notificationItemSchema, sendOnSyncDisabledWarning: notificationItemSchema, + sendOnBreakingChangeWarning: notificationItemSchema, + sendOnBreakingChangeSyncsDisabled: notificationItemSchema, }); export const notificationKeys: Array = [ @@ -215,6 +231,8 @@ export const notificationKeys: Array = [ "sendOnConnectionUpdateActionRequired", "sendOnSyncDisabled", "sendOnSyncDisabledWarning", + "sendOnBreakingChangeWarning", + "sendOnBreakingChangeSyncsDisabled", ]; export const notificationTriggerMap: Record = { @@ -224,4 +242,6 @@ export const notificationTriggerMap: Record { diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts index 0bd6eea36d3..678500667b3 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts @@ -20,6 +20,8 @@ const mockEmptyFormValues: NotificationSettingsFormValues = { sendOnConnectionUpdateActionRequired: mockNotificationItemFieldValue, sendOnSyncDisabled: mockNotificationItemFieldValue, sendOnSyncDisabledWarning: mockNotificationItemFieldValue, + sendOnBreakingChangeWarning: mockNotificationItemFieldValue, + sendOnBreakingChangeSyncsDisabled: mockNotificationItemFieldValue, }; const mockEmptyNotificationSettings: NotificationSettings = { sendOnFailure: mockNotificationItem, @@ -28,6 +30,8 @@ const mockEmptyNotificationSettings: NotificationSettings = { sendOnConnectionUpdateActionRequired: mockNotificationItem, sendOnSyncDisabled: mockNotificationItem, sendOnSyncDisabledWarning: mockNotificationItem, + sendOnBreakingChangeWarning: mockNotificationItem, + sendOnBreakingChangeSyncsDisabled: mockNotificationItem, }; describe("notificationSettingsToFormValues", () => { diff --git a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx index 3b48c6f1ed6..3b7e89f2d90 100644 --- a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx +++ b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx @@ -4,8 +4,8 @@ import { FormattedMessage } from "react-intl"; import { GAIcon } from "components/icons/GAIcon"; import { Tooltip } from "components/ui/Tooltip"; +import { useIsFCPEnabled } from "core/api/cloud"; import { ReleaseStage } from "core/request/AirbyteClient"; -import { FeatureItem, useFeature } from "core/services/features"; import { FreeTag } from "packages/cloud/components/experiments/FreeConnectorProgram"; import styles from "./ReleaseStageBadge.module.scss"; @@ -20,7 +20,7 @@ interface ReleaseStageBadgeProps { } export const ReleaseStageBadge: React.FC = ({ stage, small, tooltip = true }) => { - const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); + const fcpEnabled = useIsFCPEnabled(); if (!stage) { return null; diff --git a/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx b/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx index 70790970f45..a14cc483bd2 100644 --- a/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx +++ b/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx @@ -1,8 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; import classNames from "classnames"; -import { ConnectorCard } from "components/ConnectorCard"; - import { ConnectorIcon } from "./ConnectorIcon"; import styles from "./ConnectorIcon.story.module.scss"; @@ -27,13 +25,6 @@ export const ValidateIcons = ({ icon }: { icon: string }) => (