From fe5f344dcdc34a61c8f8d215d24ab3facf111c23 Mon Sep 17 00:00:00 2001 From: perangel Date: Wed, 1 Nov 2023 14:33:42 -0400 Subject: [PATCH] refactor: Migrate OSS PR CI workflow to ci-k8s-runner (#8876) --- .dockerignore | 9 ++-- airbyte-proxy/test.sh | 1 + .../test/utils/AcceptanceTestHarness.java | 28 +++++++++-- .../acceptance/AdvancedAcceptanceTests.java | 10 ++-- .../acceptance/AirbyteApiAcceptanceTests.java | 20 +++++--- .../test/acceptance/BasicAcceptanceTests.java | 15 +++--- .../test/acceptance/CdcAcceptanceTests.java | 18 ++++--- .../acceptance/ConnectorBuilderTests.java | 10 ++-- .../ContainerOrchestratorAcceptanceTests.java | 27 +++++----- .../acceptance/SchemaManagementTests.java | 16 +++--- .../acceptance/VersioningAcceptanceTests.java | 6 ++- airbyte-webapp/build.dockerfile | 30 ++++++++++++ airbyte-webapp/cypress/commands/api/api.ts | 14 ++++-- .../cypress/commands/api/payloads.ts | 8 +-- airbyte-webapp/cypress/commands/connection.ts | 9 ++-- .../cypress/commands/connectorBuilder.ts | 12 +++-- .../cypress/commands/destination.ts | 6 +-- airbyte-webapp/cypress/commands/source.ts | 8 +-- airbyte-webapp/cypress/cypress.config.ts | 20 +++++--- .../e2e/connection/configuration.cy.ts | 49 +++++++++++++------ .../cypress/e2e/connectorBuilder.cy.ts | 6 +-- airbyte-webapp/package.json | 4 +- airbyte-webapp/release.dockerfile | 12 +++++ airbyte-webapp/scripts/install-githooks.sh | 3 +- .../src/core/api/hooks/connections.tsx | 2 +- airbyte-webapp/test.dockerfile | 27 ++++++++++ .../KubePodProcessIntegrationTest.java | 16 +++--- settings.gradle | 9 ++-- 28 files changed, 280 insertions(+), 115 deletions(-) create mode 100644 airbyte-webapp/build.dockerfile create mode 100644 airbyte-webapp/release.dockerfile create mode 100644 airbyte-webapp/test.dockerfile diff --git a/.dockerignore b/.dockerignore index cc026d5dc6c..cf1b1cbb2a7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,11 @@ .dockerignore .git .idea -.gradle +**/.gradle **/build **/node_modules -Dockerfile.* -docker-compose*.yaml +**/build +**/Dockerfile.* +**/docker-compose*.yaml +**/.terraform +tools/bin/workflow/* diff --git a/airbyte-proxy/test.sh b/airbyte-proxy/test.sh index bb332db80d1..f267f560055 100755 --- a/airbyte-proxy/test.sh +++ b/airbyte-proxy/test.sh @@ -28,6 +28,7 @@ function start_container () { echo $CMD eval $CMD wait_for_docker $NAME-$1; + sleep 2 } function stop_container () { diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java index cab3363d0bb..52a25e0b4c1 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java @@ -10,6 +10,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.model.Network; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Resources; @@ -107,6 +109,7 @@ import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testcontainers.utility.DockerImageName; @@ -169,6 +172,10 @@ public class AcceptanceTestHarness { // https://docs.airbyte.com/understanding-airbyte/jobs/. public static final Set IN_PROGRESS_JOB_STATUSES = Set.of(JobStatus.PENDING, JobStatus.INCOMPLETE, JobStatus.RUNNING); + private static final String KUBE_PROCESS_RUNNER_HOST = java.util.Optional.ofNullable(System.getenv("KUBE_PROCESS_RUNNER_HOST")).orElse(""); + + private static final String DOCKER_NETWORK = java.util.Optional.ofNullable(System.getenv("DOCKER_NETWORK")).orElse("bridge"); + private static boolean isKube; private static boolean isMinikube; private static boolean isGke; @@ -223,12 +230,21 @@ public AcceptanceTestHarness(final AirbyteApiClient apiClient, final UUID defaul throw new RuntimeException("KUBE Flag should also be enabled if GKE flag is enabled"); } if (!isGke) { - sourcePsql = new PostgreSQLContainer(SOURCE_POSTGRES_IMAGE_NAME) - .withUsername(SOURCE_USERNAME) + // we attach the container to the appropriate network since there are environments where we use one + // other than the default + final DockerClient dockerClient = DockerClientFactory.lazyClient(); + final List dockerNetworks = dockerClient.listNetworksCmd().withNameFilter(DOCKER_NETWORK).exec(); + final Network dockerNetwork = dockerNetworks.get(0); + final org.testcontainers.containers.Network containerNetwork = + org.testcontainers.containers.Network.builder().id(dockerNetwork.getId()).build(); + sourcePsql = (PostgreSQLContainer) new PostgreSQLContainer(SOURCE_POSTGRES_IMAGE_NAME) + .withNetwork(containerNetwork); + sourcePsql.withUsername(SOURCE_USERNAME) .withPassword(SOURCE_PASSWORD); sourcePsql.start(); - destinationPsql = new PostgreSQLContainer(DESTINATION_POSTGRES_IMAGE_NAME); + destinationPsql = (PostgreSQLContainer) new PostgreSQLContainer(DESTINATION_POSTGRES_IMAGE_NAME) + .withNetwork(containerNetwork); destinationPsql.start(); } @@ -726,7 +742,8 @@ private Map localConfig(final PostgreSQLContainer psql, final boolean withSchema) { final Map dbConfig = new HashMap<>(); // don't use psql.getHost() directly since the ip we need differs depending on environment - dbConfig.put(JdbcUtils.HOST_KEY, getHostname()); + // NOTE: Use the container ip IFF we aren't on the "bridge" network + dbConfig.put(JdbcUtils.HOST_KEY, DOCKER_NETWORK.equals("bridge") ? getHostname() : psql.getHost()); if (hiddenPassword) { dbConfig.put(JdbcUtils.PASSWORD_KEY, "**********"); @@ -747,6 +764,9 @@ private Map localConfig(final PostgreSQLContainer psql, public String getHostname() { if (isKube) { + if (!KUBE_PROCESS_RUNNER_HOST.equals("")) { + return KUBE_PROCESS_RUNNER_HOST; + } if (isMinikube) { // used with minikube driver=none instance try { diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index b7064920386..f9c5442b593 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -45,8 +45,10 @@ import io.airbyte.test.utils.Asserts; import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -88,13 +90,15 @@ class AdvancedAcceptanceTests { private static AcceptanceTestHarness testHarness; private static AirbyteApiClient apiClient; private static UUID workspaceId; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); @BeforeAll static void init() throws URISyntaxException, IOException, InterruptedException, ApiException { + final URI url = new URI(AIRBYTE_SERVER_HOST); apiClient = new AirbyteApiClient( - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api")); // work in whatever default workspace is present. workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AirbyteApiAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AirbyteApiAcceptanceTests.java index 2523ea2e495..3e0c2375fc3 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AirbyteApiAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AirbyteApiAcceptanceTests.java @@ -81,12 +81,10 @@ import io.airbyte.test.utils.Asserts; import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; @@ -121,10 +119,14 @@ public class AirbyteApiAcceptanceTests { // NOTE: this is just a base64 encoding of a jwt representing a test user in some deployments. private static final String AIRBYTE_AUTH_HEADER = "eyJ1c2VyX2lkIjogImNsb3VkLWFwaSIsICJlbWFpbF92ZXJpZmllZCI6ICJ0cnVlIn0K"; private static final String AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID = "AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID"; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); + private static final String AIRBYTE_PUBLIC_API_SERVER_HOST = + Optional.ofNullable(System.getenv("AIRBYTE_PUBLIC_API_SERVER_HOST")).orElse("http://localhost:8006"); private static final String LOCAL_PUBLIC_API_SERVER_URL = "http://localhost:8006/v1"; private static final String AUTHORIZATION_HEADER = "AUTHORIZATION"; // NOTE: this is just the default airbyte/password user's basic auth header. private static final String AIRBYTE_BASIC_AUTH_HEADER = "Basic YWlyYnl0ZTpwYXNzd29yZA=="; + private static UUID workspaceId; private static final String TEST_ACTOR_NAME = "test-actor-name"; @@ -142,9 +144,10 @@ static void init() throws Exception { final boolean isGke = System.getenv().containsKey(IS_GKE); // Set up the API client. - final var underlyingApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final URI url = new URI(AIRBYTE_SERVER_HOST); + final var underlyingApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); underlyingApiClient.setRequestInterceptor(builder -> builder.setHeader(AUTHORIZATION_HEADER, AIRBYTE_BASIC_AUTH_HEADER)); if (isGke) { @@ -166,8 +169,9 @@ static void init() throws Exception { testHarness = new AcceptanceTestHarness(configApiClient, workspaceId); testHarness.ensureCleanSlate(); + final URI publicApiUrl = new URI(AIRBYTE_PUBLIC_API_SERVER_HOST); airbyteApiClient = Airbyte.builder() - .setServerURL(LOCAL_PUBLIC_API_SERVER_URL) + .setServerURL(publicApiUrl.toString()) .setSecurity(new Security() { { diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index d6333115c54..cfb7b760bf1 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -94,6 +94,7 @@ import io.airbyte.test.utils.TestConnectionCreate; import io.temporal.client.WorkflowQueryException; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; import java.time.Duration; @@ -152,6 +153,7 @@ class BasicAcceptanceTests { // NOTE: this is just a base64 encoding of a jwt representing a test user in some deployments. private static final String AIRBYTE_AUTH_HEADER = "eyJ1c2VyX2lkIjogImNsb3VkLWFwaSIsICJlbWFpbF92ZXJpZmllZCI6ICJ0cnVlIn0K"; private static final String AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID = "AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID"; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); private static final UUID POSTGRES_SOURCE_DEF_ID = UUID.fromString("decd338e-5647-4c0b-adf4-da0e75f5a750"); private static final UUID POSTGRES_DEST_DEF_ID = UUID.fromString("25c5221d-dce2-4163-ade9-739ef790f503"); public static final String IS_GKE = "IS_GKE"; @@ -192,9 +194,10 @@ static void init() throws URISyntaxException, IOException, InterruptedException, // TODO(mfsiega-airbyte): clean up and centralize the way we do config. final boolean isGke = System.getenv().containsKey(IS_GKE); // Set up the API client. - final var underlyingApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final URI url = new URI(AIRBYTE_SERVER_HOST); + final var underlyingApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); if (isGke) { underlyingApiClient.setRequestInterceptor(builder -> builder.setHeader(GATEWAY_AUTH_HEADER, AIRBYTE_AUTH_HEADER)); @@ -202,9 +205,9 @@ static void init() throws URISyntaxException, IOException, InterruptedException, apiClient = new AirbyteApiClient(underlyingApiClient); // Set up the WebBackend API client. - final var underlyingWebBackendApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final var underlyingWebBackendApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); if (isGke) { underlyingWebBackendApiClient.setRequestInterceptor(builder -> builder.setHeader(GATEWAY_AUTH_HEADER, AIRBYTE_AUTH_HEADER)); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java index 8a8b931de4b..a40db3f164a 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java @@ -46,6 +46,7 @@ import io.airbyte.test.utils.SchemaTableNamePair; import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; import java.time.Instant; @@ -129,17 +130,20 @@ record DestinationCdcRecordMatcher(JsonNode sourceRecord, Instant minUpdatedAt, private AcceptanceTestHarness testHarness; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); + @BeforeAll - static void init() throws ApiException { + static void init() throws ApiException, URISyntaxException { + final URI url = new URI(AIRBYTE_SERVER_HOST); apiClient = new AirbyteApiClient( - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api")); webBackendApi = new WebBackendApi( - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api")); // work in whatever default workspace is present. workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ConnectorBuilderTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ConnectorBuilderTests.java index 44d3f708e35..42008e5f5d6 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ConnectorBuilderTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ConnectorBuilderTests.java @@ -34,8 +34,10 @@ import io.airbyte.test.utils.Databases; import io.airbyte.test.utils.SchemaTableNamePair; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; +import java.util.Optional; import java.util.Set; import java.util.UUID; import org.junit.jupiter.api.AfterAll; @@ -64,6 +66,7 @@ public class ConnectorBuilderTests { private static final String ECHO_SERVER_IMAGE = "mendhak/http-https-echo:29"; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); private static AirbyteApiClient apiClient; private static UUID workspaceId; @@ -165,9 +168,10 @@ public class ConnectorBuilderTests { @BeforeAll static void init() throws URISyntaxException, IOException, InterruptedException, ApiException, SQLException { - final var underlyingApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final URI url = new URI(AIRBYTE_SERVER_HOST); + final var underlyingApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); apiClient = new AirbyteApiClient(underlyingApiClient); workspaceId = apiClient.getWorkspaceApi() diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java index b7c70ae3e05..8bbc959e1d6 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/ContainerOrchestratorAcceptanceTests.java @@ -27,8 +27,10 @@ import io.airbyte.test.utils.TestConnectionCreate; import io.fabric8.kubernetes.client.KubernetesClient; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.Executors; @@ -61,19 +63,22 @@ class ContainerOrchestratorAcceptanceTests { private static final Logger LOGGER = LoggerFactory.getLogger(ContainerOrchestratorAcceptanceTests.class); - private static final String AIRBYTE_WORKER = "airbyte-worker"; - private static final String DEFAULT = "default"; + private static final String AIRBYTE_WORKER = Optional.ofNullable(System.getenv("AIRBYTE_WORKER_DEPLOYMENT_NAME")).orElse("airbyte-worker"); + private static final String NAMESPACE = Optional.ofNullable(System.getenv("NAMESPACE")).orElse("default"); private static AcceptanceTestHarness testHarness; private static AirbyteApiClient apiClient; private static KubernetesClient kubernetesClient; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); + @BeforeAll static void init() throws URISyntaxException, IOException, InterruptedException, ApiException { + final URI url = new URI(AIRBYTE_SERVER_HOST); apiClient = new AirbyteApiClient( - new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api")); // work in whatever default workspace is present. UUID workspaceId = apiClient.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); @@ -136,10 +141,10 @@ void testDowntimeDuringSync() throws Exception { waitWhileJobHasStatus(apiClient.getJobsApi(), connectionSyncRead.getJob(), Set.of(JobStatus.PENDING)); LOGGER.info("Scaling down worker..."); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(0, true); LOGGER.info("Scaling up worker..."); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(1); waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); } @@ -170,8 +175,8 @@ void testCancelSyncWithInterruption() throws Exception { final JobInfoRead connectionSyncRead = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); waitWhileJobHasStatus(apiClient.getJobsApi(), connectionSyncRead.getJob(), Set.of(JobStatus.PENDING)); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(0, true); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(1); final var resp = apiClient.getJobsApi().cancelJob(new JobIdRequestBody().id(connectionSyncRead.getJob().getId())); assertEquals(JobStatus.CANCELLED, resp.getJob().getStatus()); @@ -209,7 +214,7 @@ void testCancelSyncWhenCancelledWhenWorkerIsNotRunning() throws Exception { Thread.sleep(1000); LOGGER.info("Scale down workers..."); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(0, true); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(0, true); LOGGER.info("Starting background cancellation request..."); final var pool = Executors.newSingleThreadExecutor(); @@ -229,7 +234,7 @@ void testCancelSyncWhenCancelledWhenWorkerIsNotRunning() throws Exception { Thread.sleep(2000); LOGGER.info("Scaling up workers..."); - kubernetesClient.apps().deployments().inNamespace(DEFAULT).withName(AIRBYTE_WORKER).scale(1); + kubernetesClient.apps().deployments().inNamespace(NAMESPACE).withName(AIRBYTE_WORKER).scale(1); LOGGER.info("Waiting for cancellation to go into effect..."); assertEquals(JobStatus.CANCELLED, resp.get().getJob().getStatus()); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/SchemaManagementTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/SchemaManagementTests.java index 8af8f5b55f0..1d4b4e98e69 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/SchemaManagementTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/SchemaManagementTests.java @@ -36,8 +36,10 @@ import io.airbyte.test.utils.Asserts; import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; @@ -70,6 +72,7 @@ class SchemaManagementTests { // NOTE: this is just a base64 encoding of a jwt representing a test user in some deployments. private static final String AIRBYTE_AUTH_HEADER = "eyJ1c2VyX2lkIjogImNsb3VkLWFwaSIsICJlbWFpbF92ZXJpZmllZCI6ICJ0cnVlIn0K"; private static final String AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID = "AIRBYTE_ACCEPTANCE_TEST_WORKSPACE_ID"; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); public static final int JITTER_MAX_INTERVAL_SECS = 10; public static final int FINAL_INTERVAL_SECS = 60; public static final int MAX_TRIES = 3; @@ -122,9 +125,10 @@ static void init() throws ApiException, URISyntaxException, IOException, Interru // TODO(mfsiega-airbyte): clean up and centralize the way we do config. final boolean isGke = System.getenv().containsKey(IS_GKE); // Set up the API client. - final var underlyingApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final URI url = new URI(AIRBYTE_SERVER_HOST); + final var underlyingApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); if (isGke) { underlyingApiClient.setRequestInterceptor(builder -> builder.setHeader(GATEWAY_AUTH_HEADER, AIRBYTE_AUTH_HEADER)); @@ -132,9 +136,9 @@ static void init() throws ApiException, URISyntaxException, IOException, Interru apiClient = new AirbyteApiClient(underlyingApiClient); // Set up the WebBackend API client. - final var underlyingWebBackendApiClient = new ApiClient().setScheme("http") - .setHost("localhost") - .setPort(8001) + final var underlyingWebBackendApiClient = new ApiClient().setScheme(url.getScheme()) + .setHost(url.getHost()) + .setPort(url.getPort()) .setBasePath("/api"); if (isGke) { underlyingWebBackendApiClient.setRequestInterceptor(builder -> builder.setHeader(GATEWAY_AUTH_HEADER, AIRBYTE_AUTH_HEADER)); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java index 45d34baf122..68b2342e1f7 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java @@ -20,6 +20,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.time.Duration; +import java.util.Optional; import java.util.UUID; import okhttp3.OkHttpClient; import org.junit.jupiter.api.BeforeAll; @@ -33,16 +34,17 @@ class VersioningAcceptanceTests { private static AirbyteApiClient2 apiClient2; private static UUID workspaceId; + private static final String AIRBYTE_SERVER_HOST = Optional.ofNullable(System.getenv("AIRBYTE_SERVER_HOST")).orElse("http://localhost:8001"); @BeforeAll - static void init() throws IOException { + static void init() throws IOException, URISyntaxException { RetryPolicy policy = RetryPolicy.builder() .handle(Throwable.class) .withMaxAttempts(5) .withBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)).build(); OkHttpClient client = new OkHttpClient.Builder().readTimeout(Duration.ofSeconds(20)).build(); - apiClient2 = new AirbyteApiClient2("http://localhost:8001/api", policy, client); + apiClient2 = new AirbyteApiClient2(String.format("%s/api", AIRBYTE_SERVER_HOST), policy, client); workspaceId = apiClient2.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); } diff --git a/airbyte-webapp/build.dockerfile b/airbyte-webapp/build.dockerfile new file mode 100644 index 00000000000..d8b037255e2 --- /dev/null +++ b/airbyte-webapp/build.dockerfile @@ -0,0 +1,30 @@ +FROM node:18.15.0-slim AS base + +ENV PNPM_HOME=/pnpm +ENV PATH="$PNPM_HOME:$PATH" +ENV NPM_CONFIG_PREFIX=/pnpm +ARG PNPM_VERSION=8.6.12 +ARG PNPM_STORE_DIR=/pnpm/store +ARG PROJECT_DIR + +RUN apt update && apt install -y \ + curl \ + git \ + xxd + +RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate +RUN pnpm config set store-dir $PNPM_STORE_DIR + +COPY . /workspace +WORKDIR ${PROJECT_DIR} + +FROM base AS prod-deps +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile + +FROM base AS build +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm build + +FROM base AS common +COPY --from=prod-deps $PROJECT_DIR/node_modules/ $PROJECT_DIR/node_modules +COPY --from=build $PROJECT_DIR $PROJECT_DIR/build/app diff --git a/airbyte-webapp/cypress/commands/api/api.ts b/airbyte-webapp/cypress/commands/api/api.ts index 2f984d21190..96fb0d6943a 100644 --- a/airbyte-webapp/cypress/commands/api/api.ts +++ b/airbyte-webapp/cypress/commands/api/api.ts @@ -19,7 +19,7 @@ import { import { getWorkspaceId, setWorkspaceId } from "./workspace"; -const getApiUrl = (path: string): string => `http://localhost:8001/api/v1${path}`; +const getApiUrl = (path: string): string => `${Cypress.env("AIRBYTE_SERVER_BASE_URL")}/api/v1${path}`; const apiRequest = ( method: Cypress.HttpMethod, @@ -47,7 +47,9 @@ export const completeInitialSetup = () => }); export const requestConnectionsList = () => - apiRequest("POST", "/connections/list", { workspaceId: getWorkspaceId() }); + apiRequest("POST", "/connections/list", { + workspaceId: getWorkspaceId(), + }); export const requestCreateConnection = (body: ConnectionCreate) => apiRequest("POST", "/web_backend/connections/create", body); @@ -62,7 +64,9 @@ export const requestDeleteConnection = (body: ConnectionIdRequestBody) => apiRequest("POST", "/connections/delete", body, 204); export const requestSourcesList = () => - apiRequest("POST", "/sources/list", { workspaceId: getWorkspaceId() }); + apiRequest("POST", "/sources/list", { + workspaceId: getWorkspaceId(), + }); export const requestSourceDiscoverSchema = (body: SourceDiscoverSchemaRequestBody) => apiRequest("POST", "/sources/discover_schema", body); @@ -72,7 +76,9 @@ export const requestCreateSource = (body: SourceCreate) => apiRequest apiRequest("POST", "/sources/delete", body, 204); export const requestDestinationsList = () => - apiRequest("POST", "/destinations/list", { workspaceId: getWorkspaceId() }); + apiRequest("POST", "/destinations/list", { + workspaceId: getWorkspaceId(), + }); export const requestCreateDestination = (body: DestinationCreate) => apiRequest("POST", "/destinations/create", body); diff --git a/airbyte-webapp/cypress/commands/api/payloads.ts b/airbyte-webapp/cypress/commands/api/payloads.ts index 32e0c7f0cac..791b87a534d 100644 --- a/airbyte-webapp/cypress/commands/api/payloads.ts +++ b/airbyte-webapp/cypress/commands/api/payloads.ts @@ -24,9 +24,9 @@ export const getPostgresCreateSourceBody = (name: string): SourceCreate => ({ tunnel_method: { tunnel_method: "NO_TUNNEL" }, replication_method: { method: "Standard" }, ssl: false, - port: 5433, + port: Cypress.env("SOURCE_DB_PORT") || 5433, schemas: ["public"], - host: "localhost", + host: Cypress.env("SOURCE_DB_HOST") || "localhost", database: "airbyte_ci_source", username: "postgres", password: "secret_password", @@ -54,9 +54,9 @@ export const getPostgresCreateDestinationBody = (name: string): DestinationCreat ssl_mode: { mode: "disable" }, tunnel_method: { tunnel_method: "NO_TUNNEL" }, ssl: false, - port: 5434, + port: Cypress.env("DESTINATION_DB_PORT") || 5434, schema: "public", - host: "localhost", + host: Cypress.env("DESTINATION_DB_HOST") || "localhost", database: "airbyte_ci_destination", username: "postgres", password: "secret_password", diff --git a/airbyte-webapp/cypress/commands/connection.ts b/airbyte-webapp/cypress/commands/connection.ts index 26c3c0f8d6a..b532431a432 100644 --- a/airbyte-webapp/cypress/commands/connection.ts +++ b/airbyte-webapp/cypress/commands/connection.ts @@ -1,10 +1,10 @@ import { ConnectionStatus, WebBackendConnectionRead } from "@src/core/api/generated/AirbyteClient.schemas"; -import { WebBackendConnectionCreate, DestinationRead, SourceRead } from "@src/core/api/types/AirbyteClient"; +import { DestinationRead, SourceRead, WebBackendConnectionCreate } from "@src/core/api/types/AirbyteClient"; import { + enterConnectionName, selectSchedule, setupDestinationNamespaceSourceFormat, - enterConnectionName, } from "pages/connection/connectionFormPageObject"; import { openCreateConnection } from "pages/destinationPage"; @@ -137,7 +137,10 @@ export const createNewConnectionViaApi = ( let connectionRequestBody: WebBackendConnectionCreate; const myConnection = requestWorkspaceId().then(() => { - requestSourceDiscoverSchema({ sourceId: source.sourceId, disable_cache: true }).then(({ catalog, catalogId }) => { + requestSourceDiscoverSchema({ + sourceId: source.sourceId, + disable_cache: true, + }).then(({ catalog, catalogId }) => { if (options.enableAllStreams) { catalog?.streams.forEach((stream) => { if (stream.config) { diff --git a/airbyte-webapp/cypress/commands/connectorBuilder.ts b/airbyte-webapp/cypress/commands/connectorBuilder.ts index e65d1b24d85..a14fcd7e6bf 100644 --- a/airbyte-webapp/cypress/commands/connectorBuilder.ts +++ b/airbyte-webapp/cypress/commands/connectorBuilder.ts @@ -1,12 +1,13 @@ import { addStream, assertHasNumberOfPages, - configureListStreamSlicer, configureLimitOffsetPagination, + configureListStreamSlicer, + disableAutoImportSchema, disablePagination, disableStreamSlicer, - enableStreamSlicer, enablePagination, + enableStreamSlicer, enterName, enterRecordSelector, enterStreamName, @@ -24,14 +25,15 @@ import { openTestInputs, selectAuthMethod, submitForm, - disableAutoImportSchema, } from "pages/connectorBuilderPage"; export const configureGlobals = (name: string) => { goToView("global"); enterName(name); - if (Cypress.platform === "darwin") { + if (Cypress.env("MOCK_API_SERVER_HOST")) { + enterUrlBase(Cypress.env("MOCK_API_SERVER_HOST")); + } else if (Cypress.platform === "darwin") { enterUrlBase("http://host.docker.internal:6767/"); } else { enterUrlBase("http://172.17.0.1:6767/"); @@ -79,7 +81,7 @@ export const cleanUp = () => { export const publishProject = () => { // debounce is 2500 so we need to wait at least more before change page // eslint-disable-next-line cypress/no-unnecessary-waiting - cy.wait(3000); + cy.wait(30000); cy.get('[data-testid="publish-button"]').click({ force: true }); submitForm(); }; diff --git a/airbyte-webapp/cypress/commands/destination.ts b/airbyte-webapp/cypress/commands/destination.ts index 14fba597ccf..08affb2b66b 100644 --- a/airbyte-webapp/cypress/commands/destination.ts +++ b/airbyte-webapp/cypress/commands/destination.ts @@ -12,13 +12,13 @@ export const createLocalJsonDestination = (name: string, destinationPath = "/loc fillLocalJsonForm(name, destinationPath); submitButtonClick(); - cy.wait("@checkDestinationConnection", { requestTimeout: 8000 }); + cy.wait("@checkDestinationConnection", { requestTimeout: 30000 }); cy.wait("@createDestination"); }; export const createPostgresDestination = ( name: string, - host = "localhost", + host = Cypress.env("DESTINATION_DB_HOST") || "localhost", port = "5434", database = "airbyte_ci_destination", username = "postgres", @@ -33,7 +33,7 @@ export const createPostgresDestination = ( fillPostgresForm(name, host, port, database, username, password, schema, true); submitButtonClick(); - cy.wait("@checkDestinationConnection", { requestTimeout: 8000 }); + cy.wait("@checkDestinationConnection", { requestTimeout: 30000 }); cy.wait("@createDestination"); }; diff --git a/airbyte-webapp/cypress/commands/source.ts b/airbyte-webapp/cypress/commands/source.ts index a57d6c6504e..eeef91a9ceb 100644 --- a/airbyte-webapp/cypress/commands/source.ts +++ b/airbyte-webapp/cypress/commands/source.ts @@ -1,12 +1,12 @@ import { goToSourcePage, openNewSourcePage } from "pages/sourcePage"; import { deleteEntity, openConnectorPage, submitButtonClick, updateField } from "./common"; -import { fillDummyApiForm, fillPostgresForm, fillPokeAPIForm } from "./connector"; +import { fillDummyApiForm, fillPokeAPIForm, fillPostgresForm } from "./connector"; export const createPostgresSource = ( name: string, - host = "localhost", - port = "5433", + host = Cypress.env("SOURCE_DB_HOST") || "localhost", + port = Cypress.env("SOURCE_DB_PORT") || "5433", database = "airbyte_ci_source", username = "postgres", password = "secret_password", @@ -46,7 +46,7 @@ export const createDummyApiSource = (name: string) => { fillDummyApiForm(name, "theauthkey"); submitButtonClick(); - cy.wait("@checkSourceUpdateConnection"); + cy.wait("@checkSourceUpdateConnection", { timeout: 60000 }); cy.wait("@createSource"); }; diff --git a/airbyte-webapp/cypress/cypress.config.ts b/airbyte-webapp/cypress/cypress.config.ts index 8d319eda844..c73e74ba57f 100644 --- a/airbyte-webapp/cypress/cypress.config.ts +++ b/airbyte-webapp/cypress/cypress.config.ts @@ -4,11 +4,11 @@ import pg from "pg-promise"; const pgp = pg(); export const DB_CONFIG = { - user: "postgres", - host: "127.0.0.1", - database: "airbyte_ci_source", - password: "secret_password", - port: 5433, + user: process.env.SOURCE_DB_USER || "postgres", + host: process.env.SOURCE_DB_HOST || "127.0.0.1", + database: process.env.SOURCE_DB_NAME || "airbyte_ci_source", + password: process.env.SOURCE_DB_PASSWORD || "secret_password", + port: process.env.SOURCE_DB_PORT || 5433, }; export default defineConfig({ @@ -19,10 +19,18 @@ export default defineConfig({ baseUrl: "https://localhost:3000", specPattern: ["cypress/e2e/**/*.cy.ts"], supportFile: "cypress/support/e2e.ts", - setupNodeEvents(on) { + setupNodeEvents(on, config) { on("task", { dbQuery(params) { const { query, connection = DB_CONFIG } = params; + // apply override if set + if (config.env.SOURCE_DB_HOST) { + connection.host = config.env.SOURCE_DB_HOST; + } + // apply override if set + if (config.env.SOURCE_DB_PORT) { + connection.port = config.env.SOURCE_DB_PORT; + } const db = pgp(connection); return db.any(query).finally(db.$pool.end); }, diff --git a/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts b/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts index 30dff1bd715..6df516af284 100644 --- a/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts +++ b/airbyte-webapp/cypress/e2e/connection/configuration.cy.ts @@ -8,12 +8,12 @@ import { import { appendRandomString, deleteEntity, submitButtonClick } from "@cy/commands/common"; import { createJsonDestinationViaApi, + createNewConnectionViaApi, createPokeApiSourceViaApi, createPostgresDestinationViaApi, createPostgresSourceViaApi, - createNewConnectionViaApi, - startManualSync, startManualReset, + startManualSync, } from "@cy/commands/connection"; import { runDbQuery } from "@cy/commands/db/db"; import { createUsersTableQuery, dropUsersTableQuery } from "@cy/commands/db/queries"; @@ -29,13 +29,13 @@ import { visit } from "@cy/pages/connection/connectionPageObject"; import * as replicationPage from "@cy/pages/connection/connectionReplicationPageObject"; import { streamsTable } from "@cy/pages/connection/StreamsTablePageObject"; import { - WebBackendConnectionRead, + AirbyteStreamAndConfiguration, + ConnectionStatus, DestinationRead, + DestinationSyncMode, SourceRead, SyncMode, - DestinationSyncMode, - ConnectionStatus, - AirbyteStreamAndConfiguration, + WebBackendConnectionRead, } from "@src/core/api/types/AirbyteClient"; import { ConnectionRoutePaths, RoutePaths } from "@src/pages/routePaths"; @@ -81,10 +81,14 @@ describe("Connection Configuration", () => { requestDeleteSource({ sourceId: postgresSource.sourceId }); } if (jsonDestination) { - requestDeleteDestination({ destinationId: jsonDestination.destinationId }); + requestDeleteDestination({ + destinationId: jsonDestination.destinationId, + }); } if (postgresDestination) { - requestDeleteDestination({ destinationId: postgresDestination.destinationId }); + requestDeleteDestination({ + destinationId: postgresDestination.destinationId, + }); } runDbQuery(dropUsersTableQuery); }); @@ -114,7 +118,11 @@ describe("Connection Configuration", () => { // sync & verify the button enters and exits its disabled state as the status updates startManualSync(); cy.get("[data-testid='manual-sync-button']").should("be.disabled"); - cy.get("[data-testid='connection-status-text']").contains("On time", { timeout: 20000 }).should("exist"); + cy.get("[data-testid='connection-status-text']") + .contains("On time", { + timeout: 30000, + }) + .should("exist"); cy.get("[data-testid='manual-sync-button']").should("not.be.disabled"); }); }); @@ -126,7 +134,11 @@ describe("Connection Configuration", () => { // reset & verify the button enters and exits its disabled state as the status updates startManualReset(); cy.get("[data-testid='manual-reset-button']").should("be.disabled"); - cy.get("[data-testid='connection-status-text']").contains("Pending").should("exist"); + cy.get("[data-testid='connection-status-text']") + .contains("Pending", { + timeout: 30000, + }) + .should("exist"); cy.get("[data-testid='manual-reset-button']").should("not.be.disabled"); }); }); @@ -168,7 +180,10 @@ describe("Connection Configuration", () => { const { scheduleType, scheduleData, schedule, ...connectionUpdate } = interception.response?.body; expect(scheduleType).to.eq("cron"); - expect(scheduleData.cron).to.deep.eq({ cronTimeZone: "UTC", cronExpression: "0 0 12 * * ?" }); + expect(scheduleData.cron).to.deep.eq({ + cronTimeZone: "UTC", + cronExpression: "0 0 12 * * ?", + }); expect(loadedConnection).to.deep.eq(connectionUpdate); }); replicationPage.checkSuccessResult(); @@ -422,7 +437,9 @@ describe("Connection Configuration", () => { (stream) => stream.stream?.name === "users" && stream.stream.namespace === "public" ); - const newSyncCatalog = { streams: [...postgresConnection.syncCatalog.streams] }; + const newSyncCatalog = { + streams: [...postgresConnection.syncCatalog.streams], + }; // update so one stream is enabled, to test that you can still filter by enabled/disabled streams newSyncCatalog.streams[streamToUpdate].config = { ...newSyncCatalog.streams[streamToUpdate].config, @@ -435,7 +452,9 @@ describe("Connection Configuration", () => { getPostgresToPostgresUpdateConnectionBody(postgresConnection.connectionId, { syncCatalog: newSyncCatalog }) ); - requestDeleteConnection({ connectionId: postgresConnection.connectionId }); + requestDeleteConnection({ + connectionId: postgresConnection.connectionId, + }); }); }); @@ -540,7 +559,9 @@ describe("Connection Configuration", () => { createNewConnectionViaApi(postgresSource, postgresDestination) .then((connection) => { requestUpdateConnection( - getPostgresToPostgresUpdateConnectionBody(connection.connectionId, { status: ConnectionStatus.inactive }) + getPostgresToPostgresUpdateConnectionBody(connection.connectionId, { + status: ConnectionStatus.inactive, + }) ); }) .as("postgresConnection"); diff --git a/airbyte-webapp/cypress/e2e/connectorBuilder.cy.ts b/airbyte-webapp/cypress/e2e/connectorBuilder.cy.ts index 8de95185c12..8e594f42ab1 100644 --- a/airbyte-webapp/cypress/e2e/connectorBuilder.cy.ts +++ b/airbyte-webapp/cypress/e2e/connectorBuilder.cy.ts @@ -2,16 +2,16 @@ import { appendRandomString } from "commands/common"; import { createTestConnection, startManualSync } from "commands/connection"; import { acceptSchemaWithMismatch, + assertHasNumberOfSlices, assertMaxNumberOfPages, assertMaxNumberOfSlices, assertMaxNumberOfSlicesAndPages, assertMultiPageReadItems, - assertHasNumberOfSlices, - assertTestReadItems, - assertTestReadAuthFailure, assertSchema, assertSchemaMismatch, assertSource404Error, + assertTestReadAuthFailure, + assertTestReadItems, assertUrlPath, cleanUp, configureAuth, diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index b75e0a2328f..1392c8616d9 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -38,8 +38,8 @@ "preanalyze-lowcode": "TS_NODE_TRANSPILE_ONLY=true pnpm run generate-client", "analyze-lowcode": "ts-node --skip-project ./scripts/analyze-low-code-manifests.ts", "cypress:open": "cypress open --config-file cypress/cypress.config.ts", - "cypress:ci": "CYPRESS_BASE_URL=http://localhost:8000 cypress run --config-file cypress/cypress.config.ts", - "cypress:ci:record": "CYPRESS_BASE_URL=http://localhost:8000 cypress run --config-file cypress/cypress.config.ts --record --key $CYPRESS_KEY", + "cypress:ci": "CYPRESS_BASE_URL=${CYPRESS_BASE_URL:-http://localhost:8000} CYPRESS_AIRBYTE_SERVER_BASE_URL=${CYPRESS_AIRBYTE_SERVER_BASE_URL:-http://localhost:8001} cypress run --config-file cypress/cypress.config.ts", + "cypress:ci:record": "CYPRESS_BASE_URL=${CYPRESS_BASE_URL:-http://localhost:8000} CYPRESS_AIRBYTE_SERVER_BASE_URL=${CYPRESS_AIRBYTE_SERVER_BASE_URL:-http://localhost:8001} cypress run --config-file cypress/cypress.config.ts --record --key $CYPRESS_KEY", "createdbsource": "docker run --rm -d -p 5433:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_source --name airbyte_ci_pg_source postgres", "createdbdestination": "docker run --rm -d -p 5434:5432 -e POSTGRES_PASSWORD=secret_password -e POSTGRES_DB=airbyte_ci_destination --name airbyte_ci_pg_destination postgres", "createdummyapi": "docker run --rm -d -p 6767:6767 --mount type=bind,source=\"$(pwd)\"/scripts/dummy_api.js,target=/index.js --name=dummy_api node:16-alpine \"index.js\"", diff --git a/airbyte-webapp/release.dockerfile b/airbyte-webapp/release.dockerfile new file mode 100644 index 00000000000..27a7aef30f8 --- /dev/null +++ b/airbyte-webapp/release.dockerfile @@ -0,0 +1,12 @@ +ARG BUILD_IMAGE +FROM ${BUILD_IMAGE} AS builder + + +FROM nginx:alpine AS release + +EXPOSE 80 + +ARG SRC_DIR=/workspace/oss/airbyte-webapp/build/app/build/app + +COPY --from=builder ${SRC_DIR} /usr/share/nginx/html +COPY nginx/default.conf.template /etc/nginx/templates/default.conf.template diff --git a/airbyte-webapp/scripts/install-githooks.sh b/airbyte-webapp/scripts/install-githooks.sh index c0d6b57e82f..ee303dd5191 100755 --- a/airbyte-webapp/scripts/install-githooks.sh +++ b/airbyte-webapp/scripts/install-githooks.sh @@ -1,5 +1,6 @@ -#! /bin/sh +#!/usr/bin/env bash +[[ $(git rev-parse --is-inside-work-tree > /dev/null 2>&1) ]] || exit 0 # Get the current core.hooksPath config hooksDir="$(git config core.hooksPath)" diff --git a/airbyte-webapp/src/core/api/hooks/connections.tsx b/airbyte-webapp/src/core/api/hooks/connections.tsx index e3360ce1000..89d296a2f93 100644 --- a/airbyte-webapp/src/core/api/hooks/connections.tsx +++ b/airbyte-webapp/src/core/api/hooks/connections.tsx @@ -334,7 +334,7 @@ export const useConnectionList = ( return useQuery(queryKey, queryFn, { refetchInterval: REFETCH_CONNECTION_LIST_INTERVAL, suspense: true, - }).data; + }).data as ConnectionListTransformed; }; export const useGetConnectionState = (connectionId: string) => { diff --git a/airbyte-webapp/test.dockerfile b/airbyte-webapp/test.dockerfile new file mode 100644 index 00000000000..f4ddcdf2d75 --- /dev/null +++ b/airbyte-webapp/test.dockerfile @@ -0,0 +1,27 @@ +FROM node:18.15.0-slim + +ENV PNPM_HOME=/pnpm +ENV PATH="$PNPM_HOME:$PATH" +ENV NPM_CONFIG_PREFIX=/pnpm +ARG PNPM_VERSION=8.6.12 +ARG PNPM_STORE_DIR=/pnpm/store +ARG PROJECT_DIR + +RUN apt update && apt install -y \ + curl \ + git \ + xxd + +# Install Cypress dependencies +RUN apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb + +RUN corepack enable && corepack prepare pnpm@${PNPM_VERSION} --activate +RUN pnpm config set store-dir $PNPM_STORE_DIR + +COPY . /workspace +WORKDIR ${PROJECT_DIR} + +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm generate-client +RUN pnpm cypress install diff --git a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java index ea55c3c7a54..45108e4e9b5 100644 --- a/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java +++ b/airbyte-workers/src/test-integration/java/io/airbyte/workers/process/KubePodProcessIntegrationTest.java @@ -84,8 +84,10 @@ class KubePodProcessIntegrationTest { private static final int RANDOM_FILE_LINE_LENGTH = 100; private static final boolean IS_MINIKUBE = Boolean.parseBoolean(Optional.ofNullable(System.getenv("IS_MINIKUBE")).orElse("false")); + private static final String KUBE_PROCESS_RUNNER_HOST = Optional.ofNullable(System.getenv("KUBE_PROCESS_RUNNER_HOST")).orElse(""); private static final UUID CONNECTION_ID = UUID.randomUUID(); private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final String NAMESPACE = Optional.ofNullable(System.getenv("NAMESPACE")).orElse("default"); private static List openPorts; @Value("${micronaut.server.port}") private Integer heartbeatPort; @@ -107,7 +109,7 @@ void setup() throws Exception { fabricClient = new DefaultKubernetesClient(); - processFactory = new KubeProcessFactory(getWorkerConfigProviderStub(), new TestClient(), "default", "airbyte-admin", fabricClient, + processFactory = new KubeProcessFactory(getWorkerConfigProviderStub(), new TestClient(), NAMESPACE, "airbyte-admin", fabricClient, heartbeatUrl, getHost()); } @@ -262,8 +264,8 @@ void testSuccessfulSpawningWithQuotes() throws Exception { final var availablePortsBefore = KubePortManagerSingleton.getInstance().getNumAvailablePorts(); final Process process = getProcess("echo \"h\\\"i\"; sleep 1; echo hi2"); final var output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - assertEquals("h\"i\nhi2\n", output); process.waitFor(); + assertEquals("h\"i\nhi2\n", output); // the pod should be dead and in a good state assertFalse(process.isAlive()); @@ -276,8 +278,8 @@ void testEnvMapSet() throws Exception { // start a finite process final Process process = getProcess("echo ENV_VAR_1=$ENV_VAR_1"); final var output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - assertEquals("ENV_VAR_1=ENV_VALUE_1\n", output); process.waitFor(); + assertEquals("ENV_VAR_1=ENV_VALUE_1\n", output); // the pod should be dead and in a good state assertFalse(process.isAlive()); @@ -304,6 +306,7 @@ void testDeletingPodImmediatelyAfterCompletion() throws Exception { final var availablePortsBefore = KubePortManagerSingleton.getInstance().getNumAvailablePorts(); final var uuid = UUID.randomUUID(); final Process process = getProcess(Map.of("uuid", uuid.toString()), "sleep 1 && exit 10"); + process.waitFor(); final var pod = fabricClient.pods().list().getItems().stream().filter(p -> p.getMetadata() != null && p.getMetadata().getLabels() != null) .filter(p -> p.getMetadata().getLabels().containsKey("uuid") && p.getMetadata().getLabels().get("uuid").equals(uuid.toString())) @@ -314,8 +317,6 @@ void testDeletingPodImmediatelyAfterCompletion() throws Exception { fabricClient.pods().delete(pod); }, () -> {})); - process.waitFor(); - // the pod should be dead with the correct error code assertFalse(process.isAlive()); assertEquals(availablePortsBefore, KubePortManagerSingleton.getInstance().getNumAvailablePorts()); @@ -354,7 +355,7 @@ void testKillingWithoutHeartbeat() throws Exception { fabricClient = new DefaultKubernetesClient(); - processFactory = new KubeProcessFactory(getWorkerConfigProviderStub(), new TestClient(), "default", "airbyte-admin", fabricClient, + processFactory = new KubeProcessFactory(getWorkerConfigProviderStub(), new TestClient(), NAMESPACE, "airbyte-admin", fabricClient, heartbeatUrl, getHost()); // start an infinite process @@ -470,7 +471,8 @@ private static Set getOpenPorts(final int count) { private static String getHost() { try { - return (IS_MINIKUBE ? Inet4Address.getLocalHost().getHostAddress() : "host.docker.internal"); + return !KUBE_PROCESS_RUNNER_HOST.equals("") ? KUBE_PROCESS_RUNNER_HOST + : (IS_MINIKUBE ? Inet4Address.getLocalHost().getHostAddress() : "host.docker.internal"); } catch (final UnknownHostException e) { throw new RuntimeException(e); } diff --git a/settings.gradle b/settings.gradle index 41afdc46f74..36feb11abee 100644 --- a/settings.gradle +++ b/settings.gradle @@ -56,12 +56,11 @@ buildCache { // we use a different caching mechanism for Dagger builds if (System.getenv("DAGGER") == null) { remote(com.github.burrunan.s3cache.AwsS3BuildCache) { - region = 'us-east-2' - bucket = 'airbyte-buildcache' - prefix = 'cache/' + region = 'us-west-2' + bucket = 'ab-ci-cache' + prefix = 'platform-ci-cache/' push = isCiServer - // Credentials will be taken from S3_BUILD_CACHE_... environment variables - // anonymous access will be used if environment variables are missing + enabled = System.getenv().containsKey("S3_BUILD_CACHE_ACCESS_KEY_ID") } } }