Merge branch 'main' into minio-storage-class

airbyte-connector-builder-server/src/main/java/io/airbyte/connector_builder/config/MetricClientFactory.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020-2024 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.connector_builder.config;
+
+import io.airbyte.metrics.lib.MetricClient;
+import io.airbyte.metrics.lib.MetricEmittingApps;
+import io.micronaut.context.annotation.Factory;
+import io.micronaut.context.annotation.Replaces;
+import jakarta.inject.Singleton;
+
+/**
+ * Micronaut singleton factory for the creation and initialization of the {@link MetricClient}.
+ */
+@Factory
+public class MetricClientFactory {
+
+  /**
+   * Creates and initializes the {@link MetricClient}.
+   *
+   * @return The initialized {@link MetricClient}.
+   */
+  @Singleton
+  @Replaces(MetricClient.class)
+  public MetricClient metricClient() {
+    // Initialize the metric client
+    io.airbyte.metrics.lib.MetricClientFactory.initialize(MetricEmittingApps.CONNECTOR_BUILDER_SERVER);
+    return io.airbyte.metrics.lib.MetricClientFactory.getMetricClient();
+  }
+
+}

airbyte-keycloak-setup/Dockerfile

@@ -3,10 +3,13 @@ ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.3.1
 FROM scratch as builder
 WORKDIR /app
 ADD airbyte-app.tar /app
+ADD entrypoint.sh entrypoint.sh
 
 FROM ${JDK_IMAGE} AS keycloak-setup
 WORKDIR /app
 COPY --chown=airbyte:airbyte --from=builder /app /app
+RUN chmod +x /app/entrypoint.sh
 USER airbyte:airbyte
 
+ENTRYPOINT "/app/entrypoint.sh"
 CMD ["/bin/bash", "-c", "airbyte-app/bin/airbyte-keycloak-setup"]

airbyte-keycloak-setup/build.gradle.kts

@@ -43,3 +43,12 @@ airbyte {
     imageName = "keycloak-setup"
   }
 }
+
+val copyScripts = tasks.register<Copy>("copyScripts") {
+  from("scripts")
+  into("build/airbyte/docker/")
+}
+
+tasks.named("dockerCopyDistribution") {
+  dependsOn(copyScripts)
+}

airbyte-keycloak-setup/scripts/entrypoint.sh

@@ -0,0 +1,3 @@
+#/bin/bash
+until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done;
+/app/airbyte-app/bin/airbyte-keycloak-setup

airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/MetricEmittingApps.java

@@ -31,7 +31,8 @@ public enum MetricEmittingApps implements MetricEmittingApp {
   WORKER("worker"),
   WORKLOAD_API("workload-api"),
   WORKLOAD_INIT("workload-init"),
-  WORKLOAD_LAUNCHER("workload-launcher");
+  WORKLOAD_LAUNCHER("workload-launcher"),
+  CONNECTOR_BUILDER_SERVER("connector-builder-server");
 
   private final String applicationName;
 

airbyte-pod-sweeper/Dockerfile

@@ -0,0 +1,12 @@
+FROM bitnami/kubectl:1.31.4
+
+# Create the Airbyte User
+USER root
+RUN groupadd --gid 1000 airbyte && useradd --uid 1000 -g airbyte --create-home --no-log-init airbyte
+
+USER airbyte:airbyte
+
+COPY --chown=airbyte:airbyte sweep-pod.sh /app/sweep-pod.sh
+RUN chmod +x /app/sweep-pod.sh
+
+ENTRYPOINT "/app/sweep-pod.sh"

airbyte-pod-sweeper/build.gradle.kts

@@ -0,0 +1,19 @@
+plugins {
+  id("io.airbyte.gradle.docker")
+  id("io.airbyte.gradle.publish")
+}
+
+airbyte {
+  docker {
+    imageName = "pod-sweeper"
+  }
+}
+
+val copyScripts = tasks.register<Copy>("copyScripts") {
+  from("scripts")
+  into("build/airbyte/docker/")
+}
+
+tasks.named("dockerCopyDistribution") {
+  dependsOn(copyScripts)
+}

airbyte-pod-sweeper/scripts/sweep-pod.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+get_job_pods () {
+    kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=job-pod \
+      get pods \
+      -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}'
+}
+delete_pod() {
+    printf "From status '%s' since '%s', " $2 $3
+    echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod
+}
+while :
+do
+    echo "Starting pod sweeper cycle:"
+
+    if [ -n "${RUNNING_TTL_MINUTES}" ]; then
+      # Time window for running pods
+      RUNNING_DATE_STR=`date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins`
+      RUNNING_DATE=`date -d $RUNNING_DATE_STR +%s`
+      echo "Will sweep running pods from before ${RUNNING_DATE_STR}"
+    fi
+
+    if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then
+      # Shorter time window for succeeded pods
+      SUCCESS_DATE_STR=`date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins`
+      SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s`
+      echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}"
+    fi
+
+    if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then
+      # Longer time window for unsuccessful pods (to debug)
+      NON_SUCCESS_DATE_STR=`date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins`
+      NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s`
+      echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}"
+    fi
+    (
+        IFS=$'\n'
+        for POD in `get_job_pods`; do
+            IFS=' '
+            POD_NAME=`echo $POD | cut -d " " -f 1`
+            POD_STATUS=`echo $POD | cut -d " " -f 2`
+            POD_DATE_STR=`echo $POD | cut -d " " -f 3`
+            POD_START_DATE_STR=`echo $POD | cut -d " " -f 4`
+            POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'`
+            if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then
+              if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then
+                  delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+              fi
+            elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Succeeded" ]; then
+              if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then
+                  delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+              fi
+            elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then
+              if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then
+                  delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+              fi
+            fi
+        done
+    )
+    echo "Completed pod sweeper cycle.  Sleeping for 60 seconds..."
+    sleep 60
+done

charts/airbyte-bootloader/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-bootloader/values.yaml

@@ -23,7 +23,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 

charts/airbyte-connector-builder-server/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-connector-builder-server/values.yaml

@@ -19,7 +19,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 enabled: true

charts/airbyte-connector-rollout-worker/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-connector-rollout-worker/values.yaml

@@ -18,7 +18,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
   enterprise:
@@ -76,20 +76,6 @@ global:
       ##  jobs.kube.main_container_image_pull_secret [string]  image pull secret to use for job pod
       main_container_image_pull_secret: ""
 
-      images:
-        ## JOB_KUBE_BUSYBOX_IMAGE
-        ## busybox image used by the job pod
-        ##   jobs.kube.images.busybox [string] busybox image used by the job pod
-        busybox: ""
-        ## JOB_KUBE_SOCAT_IMAGE
-        ## socat image used by the job pod
-        ##   jobs.kube.images.socat [string] socat image used by the job pod
-        socat: ""
-        ## JOB_KUBE_CURL_IMAGE
-        ## curl image used by the job pod
-        ##   jobs.kube.images.curl [string] curl image used by the job pod
-        curl: ""
-
 enabled: true
 ##  connector-rollout-worker.replicaCount Number of connector-rollout-worker replicas
 replicaCount: 1

charts/airbyte-cron/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-cron/values.yaml

@@ -18,7 +18,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 enabled: true

charts/airbyte-featureflag-server/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-featureflag-server/values.yaml

@@ -19,7 +19,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
   enterprise:

charts/airbyte-keycloak-setup/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-keycloak-setup/templates/job.yaml

@@ -38,17 +38,6 @@ spec:
       affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 12 }}
       {{- end }}
       initContainers:
-        - name: keycloak-readiness-check
-          image: {{ include "imageUrl" (list .Values.initContainers.keycloakReadinessCheck.image $) }}
-          command: [ "sh", "-c", "until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done;" ]
-          env:
-            - name: KEYCLOAK_INTERNAL_HOST
-              valueFrom:
-                configMapKeyRef:
-                  name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }}
-                  key: KEYCLOAK_INTERNAL_HOST
-          securityContext: 
-            {{- toYaml .Values.initContainerSecurityContext | nindent 14 }}
         {{- if .Values.extraInitContainers }}
         {{- toYaml .Values.extraInitContainers | nindent 8 }}
         {{- end }}

charts/airbyte-keycloak-setup/values.yaml

@@ -24,7 +24,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 

charts/airbyte-keycloak/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object ( connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-keycloak/values.yaml

@@ -26,7 +26,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 

charts/airbyte-metrics/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}

charts/airbyte-metrics/values.yaml

@@ -14,7 +14,7 @@ global:
     # Docker registry to pull platform images from, e.g. http://my-registry:8000/
     registry: ""
     # Image tag to use for airbyte images. 
-    # Does not include non-airbyte images such as busybox, temporal, minio, etc.
+    # Does not include non-airbyte images such as temporal, minio, etc.
     tag: ""
 
 enabled: true

charts/airbyte-pod-sweeper/templates/_images.tpl

@@ -6,7 +6,7 @@
 {{/* ensure the registry has a trailing slash, if set */}}
 {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}}
 
-{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}}
+{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}}
 {{- if (eq (typeOf $img) "string") -}}
 {{- printf "%s%s" $reg (tpl $img $root) | quote -}}
 {{- else -}}