feat: STLS bootstrap integration with linux CSE

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -116,9 +116,8 @@ HTTP_PROXY_URLS="{{GetHTTPProxy}}"
 HTTPS_PROXY_URLS="{{GetHTTPSProxy}}"
 NO_PROXY_URLS="{{GetNoProxy}}"
 PROXY_VARS="{{GetProxyVariables}}"
-ENABLE_TLS_BOOTSTRAPPING="{{EnableTLSBootstrapping}}"
 ENABLE_SECURE_TLS_BOOTSTRAPPING="{{EnableSecureTLSBootstrapping}}"
-CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID="{{GetCustomSecureTLSBootstrapAADServerAppID}}"
+CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE="{{GetCustomSecureTLSBootstrapAADResource}}"
 ENABLE_KUBELET_SERVING_CERTIFICATE_ROTATION="{{EnableKubeletServingCertificateRotation}}"
 DHCPV6_SERVICE_FILEPATH="{{GetDHCPv6ServiceCSEScriptFilepath}}"
 DHCPV6_CONFIG_FILEPATH="{{GetDHCPv6ConfigCSEScriptFilepath}}"

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -16,6 +16,7 @@ Environment="KUBE_API_SERVER_NAME=${API_SERVER_NAME}"
 EOF
   systemctlEnableAndStart reconcile-private-hosts || exit $ERR_SYSTEMCTL_START_FAIL
 }
+
 configureTransparentHugePage() {
     ETC_SYSFS_CONF="/etc/sysfs.conf"
     if [[ "${THP_ENABLED}" != "" ]]; then
@@ -164,36 +165,24 @@ EOF
   systemctl restart containerd
 }
 
-configureK8s() {
-    mkdir -p "/etc/kubernetes/certs"
-
-    APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
-    touch "${APISERVER_PUBLIC_KEY_PATH}"
-    chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
-    chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
+configureAzureJson() {
+    mkdir -p "/etc/kubernetes/"
 
     AZURE_JSON_PATH="/etc/kubernetes/azure.json"
     touch "${AZURE_JSON_PATH}"
     chmod 0600 "${AZURE_JSON_PATH}"
     chown root:root "${AZURE_JSON_PATH}"
-
+    
     set +x
-    if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
-        echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
-    fi
-    if [ -n "${KUBELET_CLIENT_CERT_CONTENT}" ]; then
-        echo "${KUBELET_CLIENT_CERT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.crt
-    fi
     if [ -n "${SERVICE_PRINCIPAL_FILE_CONTENT}" ]; then
-        echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
+        SP_FILE="/etc/kubernetes/sp.txt"
+        echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > "$SP_FILE"
+        SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"
+        SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\\/\\\\}
+        SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\"/\\\"}
+        rm "$SP_FILE"
     fi
-
-    echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
-    SP_FILE="/etc/kubernetes/sp.txt"
-    SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"
-    SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\\/\\\\}
-    SERVICE_PRINCIPAL_CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET//\"/\\\"}
-    rm "$SP_FILE"
+
     cat << EOF > "${AZURE_JSON_PATH}"
 {
     "cloud": "${TARGET_CLOUD}",
@@ -235,6 +224,42 @@ configureK8s() {
 }
 EOF
     set -x
+}
+
+ensureKubeCAFile() {
+    KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt"
+    mkdir -p "$(dirname "${KUBE_CA_FILE}")"
+    echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}"
+    chmod 0644 "${KUBE_CA_FILE}"
+}
+
+configureK8s() {
+    mkdir -p "/etc/kubernetes/certs"
+
+    APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt"
+    touch "${APISERVER_PUBLIC_KEY_PATH}"
+    chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
+    chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
+
+    if [ -n "${APISERVER_PUBLIC_KEY}" ]; then
+        set +x
+        echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
+        set -x
+    fi
+
+    if [ "$ENABLE_SECURE_TLS_BOOTSTRAPPING" == "false" ] && [ -z "$TLS_BOOTSTRAP_TOKEN" ]; then
+        # we guard the following cert creation logic since it seems RP always gives agentbaker a kubelet client cert/key pair
+        # regardless of whether the node is being bootstrapped with some flavor of TLS bootstrapping
+        set +x
+        if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
+            echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
+        fi
+        if [ -n "${KUBELET_CLIENT_CERT_CONTENT}" ]; then
+            echo "${KUBELET_CLIENT_CERT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.crt
+        fi
+        set -x
+    fi
+
     if [[ "${CLOUDPROVIDER_BACKOFF_MODE}" = "v2" ]]; then
         sed -i "/cloudProviderBackoffExponent/d" /etc/kubernetes/azure.json
         sed -i "/cloudProviderBackoffJitter/d" /etc/kubernetes/azure.json
@@ -421,6 +446,47 @@ getPrimaryNicIP() {
     echo "$ip"
 }
 
+configureSecureTLSBootstrap() {
+    # default AAD resource here so we can minimze bootstrap contract surface
+    AAD_RESOURCE="6dae42f8-4368-4678-94ff-3960e28e3630"
+    if [ -n "$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE" ]; then
+        AAD_RESOURCE="$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE"
+    fi
+
+    SECURE_TLS_BOOTSTRAPPING_DROP_IN="/etc/systemd/system/secure-tls-bootstrap.service.d/10-securetlsbootstrap.conf"
+    touch "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}"
+    chmod 0600 "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}"
+
+    cat > "${SECURE_TLS_BOOTSTRAPPING_DROP_IN}" <<EOF
+[Service]
+Environment="API_SERVER_NAME=${API_SERVER_NAME}"
+Environment="AAD_RESOURCE=${AAD_RESOURCE}"
+EOF
+}
+
+ensureSecureTLSBootstrap() {
+    KUBECONFIG_FILE=/var/lib/kubelet/kubeconfig
+
+    SECURE_TLS_BOOTSTRAP_STATUS="$(systemctl is-active secure-tls-bootstrap)"    
+    while [ "$SECURE_TLS_BOOTSTRAP_STATUS" == "activating" ]; do
+        echo "secure TLS bootstrapping is in-progress, waiting for terminal state..."
+        sleep 0.5
+        SECURE_TLS_BOOTSTRAP_STATUS="$(systemctl is-active secure-tls-bootstrap)"    
+    done
+
+    if [ "$SECURE_TLS_BOOTSTRAP_STATUS" == "failed" ] || [ "$SECURE_TLS_BOOTSTRAP_STATUS" == "is-failed" ]; then
+        systemctl status secure-tls-bootstrap --no-pager -l
+        journalctl -u secure-tls-bootstrap
+        exit $ERR_SECURE_TLS_BOOTSTRAP_CLIENT_FAIL # exit hard here until ready for preview
+    fi
+
+    if [ ! -f "$KUBECONFIG_FILE" ]; then
+        systemctl status secure-tls-bootstrap --no-pager -l
+        journalctl -u secure-tls-bootstrap
+        exit $ERR_SECURE_TLS_BOOTSTRAP_MISSING_KUBECONFIG # exit hard here until ready for preview
+    fi
+}
+
 generateSelfSignedKubeletServingCertificate() {
     mkdir -p "/etc/kubernetes/certs"
 
@@ -512,66 +578,28 @@ ensureKubelet() {
     if [ -n "${AZURE_ENVIRONMENT_FILEPATH}" ]; then
         echo "AZURE_ENVIRONMENT_FILEPATH=${AZURE_ENVIRONMENT_FILEPATH}" >> "${KUBELET_DEFAULT_FILE}"
     fi
-    chmod 0600 "${KUBELET_DEFAULT_FILE}"
-
-    KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt"
-    mkdir -p "$(dirname "${KUBE_CA_FILE}")"
-    echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}"
-    chmod 0600 "${KUBE_CA_FILE}"
 
-    if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ] || [ "${ENABLE_TLS_BOOTSTRAPPING}" == "true" ]; then
+    KUBECONFIG_FILE=/var/lib/kubelet/kubeconfig
+
+    if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ] || [ -n "$TLS_BOOTSTRAP_TOKEN" ]; then
         KUBELET_TLS_DROP_IN="/etc/systemd/system/kubelet.service.d/10-tlsbootstrap.conf"
         mkdir -p "$(dirname "${KUBELET_TLS_DROP_IN}")"
         touch "${KUBELET_TLS_DROP_IN}"
         chmod 0600 "${KUBELET_TLS_DROP_IN}"
-        tee "${KUBELET_TLS_DROP_IN}" > /dev/null <<EOF
+
+        if [ ! -f "$KUBECONFIG_FILE" ]; then
+            # if we don't have a kubelet config file, meaning that we are either bootstrapping with vanilla TLS bootstrapping
+            # or with secure TLS bootstrapping but the bootstrapping process failed, then specify the bootstrap-kubeconfig file
+            # so kubelet can request its own certificate
+            tee "${KUBELET_TLS_DROP_IN}" > /dev/null <<EOF
 [Service]
 Environment="KUBELET_TLS_BOOTSTRAP_FLAGS=--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig"
 EOF
-    fi
-
-    if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then
-        AAD_RESOURCE="6dae42f8-4368-4678-94ff-3960e28e3630"
-        if [ -n "$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID" ]; then
-            AAD_RESOURCE=$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID
-        fi
-        SECURE_BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig
-        mkdir -p "$(dirname "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}")"
-        touch "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}"
-        chmod 0644 "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}"
-        tee "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}" > /dev/null <<EOF
-apiVersion: v1
-kind: Config
-clusters:
-- name: localcluster
-  cluster:
-    certificate-authority: /etc/kubernetes/certs/ca.crt
-    server: https://${API_SERVER_NAME}:443
-users:
-- name: kubelet-bootstrap
-  user:
-    exec:
-        apiVersion: client.authentication.k8s.io/v1
-        command: /opt/azure/tlsbootstrap/tls-bootstrap-client
-        args:
-        - bootstrap
-        - --next-proto=aks-tls-bootstrap
-        - --aad-resource=${AAD_RESOURCE}
-        interactiveMode: Never
-        provideClusterInfo: true
-contexts:
-- context:
-    cluster: localcluster
-    user: kubelet-bootstrap
-  name: bootstrap-context
-current-context: bootstrap-context
-EOF
-    elif [ "${ENABLE_TLS_BOOTSTRAPPING}" == "true" ]; then
-        BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig
-        mkdir -p "$(dirname "${BOOTSTRAP_KUBECONFIG_FILE}")"
-        touch "${BOOTSTRAP_KUBECONFIG_FILE}"
-        chmod 0644 "${BOOTSTRAP_KUBECONFIG_FILE}"
-        tee "${BOOTSTRAP_KUBECONFIG_FILE}" > /dev/null <<EOF
+            BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig
+            mkdir -p "$(dirname "${BOOTSTRAP_KUBECONFIG_FILE}")"
+            touch "${BOOTSTRAP_KUBECONFIG_FILE}"
+            chmod 0644 "${BOOTSTRAP_KUBECONFIG_FILE}"
+            tee "${BOOTSTRAP_KUBECONFIG_FILE}" > /dev/null <<EOF
 apiVersion: v1
 kind: Config
 clusters:
@@ -590,8 +618,15 @@ contexts:
   name: bootstrap-context
 current-context: bootstrap-context
 EOF
+        else
+            # otherwise, if we already have a kubeconfig we can use, omit the bootstrap-kubeconfig flag
+            # so kubelet doesn't try to request its own certificate
+            tee "${KUBELET_TLS_DROP_IN}" > /dev/null <<EOF
+[Service]
+Environment="KUBELET_TLS_BOOTSTRAP_FLAGS=--kubeconfig /var/lib/kubelet/kubeconfig"
+EOF
+        fi
     else
-        KUBECONFIG_FILE=/var/lib/kubelet/kubeconfig
         mkdir -p "$(dirname "${KUBECONFIG_FILE}")"
         touch "${KUBECONFIG_FILE}"
         chmod 0644 "${KUBECONFIG_FILE}"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -84,7 +84,9 @@ ERR_ARTIFACT_STREAMING_INSTALL=153 # Error installing mirror proxy and overlaybd
 
 ERR_HTTP_PROXY_CA_CONVERT=160 # Error converting http proxy ca cert from pem to crt format
 ERR_UPDATE_CA_CERTS=161 # Error updating ca certs to include user-provided certificates
-ERR_DOWNLOAD_SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_TIMEOUT=169 # Timeout waiting for secure TLS bootrstrap kubelet exec plugin download
+ERR_SECURE_TLS_BOOTSTRAP_CLIENT_FAIL=167 # Error generating kubelet client credential via the secure TLS bootstrap client
+ERR_SECURE_TLS_BOOTSTRAP_MISSING_KUBECONFIG=168 # Unable to find kubeconfig after completion of secure TLS bootstrapping
+ERR_DOWNLOAD_SECURE_TLS_BOOTSTRAP_CLIENT=169 # Error downloading the secure TLS bootstrapping client
 
 ERR_DISBALE_IPTABLES=170 # Error disabling iptables service
 
@@ -372,6 +374,20 @@ systemctl_restart() {
         fi
     done
 }
+systemctl_restart_noblock() {
+    retries=$1; wait_sleep=$2; timeout=$3 svcname=$4
+    for i in $(seq 1 $retries); do
+        timeout $timeout systemctl daemon-reload
+        timeout $timeout systemctl restart $svcname --no-block && break || \
+        if [ $i -eq $retries ]; then
+            return 1
+        else
+            systemctl status $svcname --no-pager -l
+            journalctl -u $svcname
+            sleep $wait_sleep
+        fi
+    done
+}
 systemctl_stop() {
     retries=$1; wait_sleep=$2; timeout=$3 svcname=$4
     for i in $(seq 1 $retries); do
@@ -425,6 +441,20 @@ systemctlEnableAndStart() {
     fi
 }
 
+systemctlEnableAndStartNoBlock() {
+    systemctl_restart_noblock 100 5 30 $1
+    RESTART_STATUS=$?
+    systemctl status $1 --no-pager -l > /var/log/azure/$1-status.log
+    if [ $RESTART_STATUS -ne 0 ]; then
+        echo "$1 could not be started"
+        return 1
+    fi
+    if ! retrycmd_if_failure 120 5 25 systemctl enable --no-block $1; then
+        echo "$1 could not be enabled by systemctl"
+        return 1
+    fi
+}
+
 systemctlDisableAndStop() {
     if systemctl list-units --full --all | grep -q "$1.service"; then
         systemctl_stop 20 5 25 $1 || echo "$1 could not be stopped"

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -16,8 +16,7 @@ K8S_REGISTRY_REPO="oss/binaries/kubernetes"
 UBUNTU_RELEASE=$(lsb_release -r -s)
 # For Mariner 2.0, this returns "MARINER" and for AzureLinux 3.0, this returns "AZURELINUX"
 OS=$(if ls /etc/*-release 1> /dev/null 2>&1; then sort -r /etc/*-release | gawk 'match($0, /^(ID_LIKE=(coreos)|ID=(.*))$/, a) { print toupper(a[2] a[3]); exit }'; fi)
-SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR="/opt/azure/tlsbootstrap"
-SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION="v0.1.0-alpha.2"
+SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR="/opt/azure/containers/aks-secure-tls-bootstrap-client"
 TELEPORTD_PLUGIN_DOWNLOAD_DIR="/opt/teleportd/downloads"
 CREDENTIAL_PROVIDER_DOWNLOAD_DIR="/opt/credentialprovider/downloads"
 CREDENTIAL_PROVIDER_BIN_DIR="/var/lib/kubelet/credential-provider"
@@ -193,19 +192,17 @@ installCredentialProvider() {
     rm -rf ${CREDENTIAL_PROVIDER_DOWNLOAD_DIR}
 }
 
-downloadSecureTLSBootstrapKubeletExecPlugin() {
-    local plugin_url="https://k8sreleases.blob.core.windows.net/aks-tls-bootstrap-client/${SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION}/linux/amd64/tls-bootstrap-client"
+downloadSecureTLSBootstrapClient() {
+    # TODO(cameissner): move to components.json
+    local client_version="v0.1.0-alpha.5"
+    local client_download_url="https://stlsbootstrap.z22.web.core.windows.net/client/linux/amd64/${client_version}"
     if [[ $(isARM64) == 1 ]]; then
-        plugin_url="https://k8sreleases.blob.core.windows.net/aks-tls-bootstrap-client/${SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_VERSION}/linux/arm64/tls-bootstrap-client"
+        client_download_url="https://stlsbootstrap.z22.web.core.windows.net/client/linux/arm64/${client_version}"
     fi
-
-    mkdir -p $SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR
-    plugin_download_path="${SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR}/tls-bootstrap-client"
-
-    if [ ! -f "$plugin_download_path" ]; then
-        retrycmd_if_failure 30 5 60 curl -fSL -o "$plugin_download_path" "$plugin_url" || exit $ERR_DOWNLOAD_SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_TIMEOUT
-        chown -R root:root "$SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR"
-        chmod -R 755 "$SECURE_TLS_BOOTSTRAP_KUBELET_EXEC_PLUGIN_DOWNLOAD_DIR"
+
+    mkdir -p "/opt/azure/containers"
+    if ! retrycmd_if_failure 30 5 60 curl -fSL -o "$SECURE_TLS_BOOTSTRAP_CLIENT_BIN_DIR" "$client_download_url"; then
+        exit $ERR_DOWNLOAD_SECURE_TLS_BOOTSTRAP_CLIENT
     fi
 }
 

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -52,6 +52,17 @@ source "${CSE_INSTALL_FILEPATH}"
 source "${CSE_DISTRO_INSTALL_FILEPATH}"
 source "${CSE_CONFIG_FILEPATH}"
 
+logs_to_events "AKS.CSE.ensureKubeCAFile" ensureKubeCAFile
+
+logs_to_events "AKS.CSE.configureAzureJson" configureAzureJson
+
+if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then
+    # for now we download it so we don't have to cache it on the VHD, once it's always cached we won't need this step
+    logs_to_events "AKS.CSE.downloadSecureTLSBootstrapClient" downloadSecureTLSBootstrapClient
+    logs_to_events "AKS.CSE.configureSecureTLSBootstrap" configureSecureTLSBootstrap
+    logs_to_events "AKS.CSE.start.SecureTLSBootstrap" systemctlEnableAndStartNoBlock secure-tls-bootstrap
+fi
+
 if [[ "${DISABLE_SSH}" == "true" ]]; then
     disableSSH || exit $ERR_DISABLE_SSH
 fi
@@ -149,10 +160,6 @@ if [ "${IS_KRUSTLET}" == "true" ]; then
     logs_to_events "AKS.CSE.installSpinKube" installSpinKube "$downloadURSpinKube" "$downloadLocationSpinKube" "$versionsSpinKube"
 fi
 
-if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then
-    logs_to_events "AKS.CSE.downloadSecureTLSBootstrapKubeletExecPlugin" downloadSecureTLSBootstrapKubeletExecPlugin
-fi
-
 # By default, never reboot new nodes.
 REBOOTREQUIRED=false
 
@@ -329,7 +336,12 @@ if [ "${NEEDS_CONTAINERD}" == "true" ] &&  [ "${SHOULD_CONFIG_CONTAINERD_ULIMITS
   logs_to_events "AKS.CSE.setContainerdUlimits" configureContainerdUlimits
 fi
 
+if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then
+    logs_to_events "AKS.CSE.ensureSecureTLSBootstrap" ensureSecureTLSBootstrap
+fi
+
 logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet
+
 if [ "${ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE}" == "true" ]; then
     logs_to_events "AKS.CSE.ensureNoDupOnPromiscuBridge" ensureNoDupOnPromiscuBridge
 fi

parts/linux/cloud-init/artifacts/secure-tls-bootstrap.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs the secure TLS bootstrapping client binary to generate a kubelet client credential
+Wants=network-online.target
+After=network-online.target
+Before=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/azure/containers/secure-tls-bootstrap.sh
+RemainAfterExit=No