chore: conflicts

.pipelines/e2e.yaml

@@ -15,6 +15,7 @@ pr:
     - e2e/windows
     - pkg/agent/datamodel/sig_config*.go # SIG config changes
     - pkg/agent/datamodel/*.json # SIG version changes
+    - pkg/agent/testdata/AKSWindows* # Windows test data
 
 variables:
 - group: ab-e2e

CODEOWNERS

@@ -1 +1 @@
-* @juan-lee @cameronmeissner @UtheMan @ganeshkumarashok @anujmaheshwari1 @AlisonB319 @Devinwong @lilypan26 @ShiqianTao @AbelHu @junjiezhang1997
+* @juan-lee @cameronmeissner @UtheMan @ganeshkumarashok @anujmaheshwari1 @AlisonB319 @Devinwong @lilypan26 @ShiqianTao @AbelHu @junjiezhang1997 @jason1028kr

e2e/scenario/scenario_marinerv2.go

@@ -16,6 +16,9 @@ func (t *Template) marinerv2() *Scenario {
 				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-cblmariner-v2-gen2"
 				nbc.AgentPoolProfile.Distro = "aks-cblmariner-v2-gen2"
 			},
+			LiveVMValidators: []*LiveVMValidator{
+				KubenetEnsureNoDupEbtablesValidator(),
+			},
 		},
 	}
 }

e2e/scenario/scenario_ubuntu2204.go

@@ -16,6 +16,9 @@ func (t *Template) ubuntu2204() *Scenario {
 				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-22.04-gen2"
 				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-22.04-gen2"
 			},
+			LiveVMValidators: []*LiveVMValidator{
+				KubenetEnsureNoDupEbtablesValidator(),
+			},
 		},
 	}
 }

e2e/scenario/validators.go

@@ -2,6 +2,7 @@ package scenario
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 )
 
@@ -126,6 +127,42 @@ func UlimitValidator(ulimits map[string]string) *LiveVMValidator {
 	}
 }
 
+// KubenetEnsureNoDupEbtablesValidator checks that ebtables rules were installed by
+// the ensure-no-dup.sh script to block duplicate packets from the promiscuous bridge.
+// This assumes at least one pod (without hostNetwork) has already run on the node.
+func KubenetEnsureNoDupEbtablesValidator() *LiveVMValidator {
+	// Use regex match for the rules because the MAC and IP addresses can vary.
+	expectedRulePatterns := []string{
+		`-j AKS-DEDUP-PROMISC`,
+		`-p IPv4 -s [0-9a-f:]+ -o veth\+ --ip-src [0-9.]+ -j ACCEPT`,
+		`-p IPv4 -s [0-9a-f:]+ -o veth\+ --ip-src [0-9.]+/[0-9]+ -j DROP`,
+	}
+	regexes := make(map[string]*regexp.Regexp, len(expectedRulePatterns))
+	for _, s := range expectedRulePatterns {
+		regexes[s] = regexp.MustCompile(s)
+	}
+
+	return &LiveVMValidator{
+		Description: "assert kubenet ensure-no-dup ebtables rules",
+		// Grep matches rules with "-" at start of line.
+		// This command will fail and be retried to account for delay between
+		// when the CNI creates the bridge and when the ensure-no-dup systemd unit completes.
+		Command: fmt.Sprintf(`ebtables -L | grep "^-"`),
+		Asserter: func(code, stdout, stderr string) error {
+			if code != "0" {
+				return fmt.Errorf("validator command terminated with exit code %q but expected code 0", code)
+			}
+
+			for pattern, re := range regexes {
+				if !re.MatchString(stdout) {
+					return fmt.Errorf("could not find expected ebtables rule matching pattern %q", pattern)
+				}
+			}
+			return nil
+		},
+	}
+}
+
 func containerdVersionValidator(version string) *LiveVMValidator {
 	return &LiveVMValidator{
 		Description: "assert containerd version",

parts/linux/cloud-init/artifacts/aks-check-network.sh

@@ -6,21 +6,20 @@
 # and log the results to the events directory. For now, this script has to be triggered manually to
 # collect the log. In the future, we will run it periodically to check and alert any issue.
 
-APISERVER_FQDN=${1:-''}
-CUSTOM_ENDPOINT=${2:-''}
+CUSTOM_ENDPOINT=${1:-''}
 
 EVENTS_LOGGING_PATH="/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/"
 AZURE_CONFIG_PATH="/etc/kubernetes/azure.json"
 AKS_CA_CERT_PATH="/etc/kubernetes/certs/apiserver.crt"
 AKS_CERT_PATH="/etc/kubernetes/certs/client.crt"
 AKS_KEY_PATH="/etc/kubernetes/certs/client.key"
+AKS_KUBECONFIG_PATH="/var/lib/kubelet/kubeconfig"
 RESOLV_CONFIG_PATH="/etc/resolv.conf"
 SYSTEMD_RESOLV_CONFIG_PATH="/run/systemd/resolve/resolv.conf"
 
 ARM_ENDPOINT="management.azure.com"
 METADATA_ENDPOINT="http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://${ARM_ENDPOINT}/"
 AKS_ENDPOINT="https://${ARM_ENDPOINT}/providers/Microsoft.ContainerService/operations?api-version=2023-11-01"
-APISERVER_ENDPOINT="https://${APISERVER_FQDN}/healthz"
 
 TEMP_DIR=$(mktemp -d)
 NSLOOKUP_FILE="${TEMP_DIR}/nslookup.log"
@@ -85,9 +84,9 @@ function check_and_curl {
     # check DNS 
     nslookup $url > /dev/null
     if [ $? -eq 0 ]; then
-        logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $url'"
+        logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $url'"
     else
-        logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $url. $error_msg'"
+        logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $url. $error_msg'"
         dns_trace $url
         return 1
     fi
@@ -99,17 +98,17 @@ function check_and_curl {
         response=$(curl -s -m $MAX_TIME -o /dev/null -w "%{http_code}" "https://${url}" -L)
 
         if [ $response -ge 200 ] && [ $response -lt 400 ]; then
-            logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $url with returned status code $response'"
+            logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $url with returned status code $response'"
             break
         elif [ $response -eq 400 ] && ([ $url == "acs-mirror.azureedge.net" ] || [ $url == "eastus.data.mcr.microsoft.com" ]); then
-            logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $url with returned status code $response. This is expected since $url is a repository endpoint which requires a full package path to get 200 status code.'"
+            logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $url with returned status code $response. This is expected since $url is a repository endpoint which requires a full package path to get 200 status code.'"
             break
         else
             # if the response code is not within successful range, increment the error count
             i=$(( $i + 1 ))
             # if we have reached the maximum number of retries, log an error
             if [[ $i -eq $MAX_RETRY ]]; then
-                logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to curl $url after $MAX_RETRY attempts with returned status code $response. $error_msg'" 
+                logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to curl $url after $MAX_RETRY attempts with returned status code $response. $error_msg'" 
                 break
             fi
 
@@ -119,29 +118,30 @@ function check_and_curl {
     done
 }
 
+logs_to_events "AKS.testingTraffic.start" "echo '$(date) - INFO: Starting network connectivity check'"
 
-if ! [ -e "${AZURE_CONFIG_PATH}" ]; then
-    logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - WARNING: Failed to find $AZURE_CONFIG_PATH file. Are you running inside Kubernetes?'"
+if ! [ -f "${AZURE_CONFIG_PATH}" ]; then
+    logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - WARNING: Failed to find $AZURE_CONFIG_PATH file. Are you running inside Kubernetes?'"
 fi
 
 # check DNS resolution to ARM endpoint
 nslookup $ARM_ENDPOINT > $NSLOOKUP_FILE
 if [ $? -eq 0 ]; then
-    logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $ARM_ENDPOINT'"
+    logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $ARM_ENDPOINT'"
 else
     error_log=$(cat $NSLOOKUP_FILE)
-    logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $ARM_ENDPOINT with error $error_log'"
+    logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $ARM_ENDPOINT with error $error_log'"
 
     # check resolv.conf
     nameserver=$(cat $NSLOOKUP_FILE | grep "Server" | awk '{print $2}')
     echo "Checking resolv.conf for nameserver $nameserver"
     cat $RESOLV_CONFIG_PATH | grep $nameserver 
     if [ $? -ne 0 ]; then
-        logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - FAILURE: Nameserver $nameserver wasn't found in $RESOLV_CONFIG_PATH'"
+        logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - FAILURE: Nameserver $nameserver wasn't found in $RESOLV_CONFIG_PATH'"
     fi
     cat $SYSTEMD_RESOLV_CONFIG_PATH | grep $nameserver 
     if [ $? -ne 0 ]; then
-        logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - FAILURE: Nameserver $nameserver wasn't found in $SYSTEMD_RESOLV_CONFIG_PATH'"
+        logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - FAILURE: Nameserver $nameserver wasn't found in $SYSTEMD_RESOLV_CONFIG_PATH'"
     fi
 
     # trace request
@@ -153,33 +153,35 @@ fi
 # check access to ARM endpoint
 result=$(curl -m $MAX_TIME -s -o $TOKEN_FILE -w "%{http_code}" -H Metadata:true $METADATA_ENDPOINT)
 if [ $result -eq 200 ]; then
-    logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully retrieved access token'"
+    logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully retrieved access token'"
     access_token=$(cat $TOKEN_FILE | jq -r .access_token)
     res=$(curl -m $MAX_TIME -X GET -H "Authorization: Bearer $access_token" -H "Content-Type:application/json" -s -o /dev/null -w "%{http_code}" $AKS_ENDPOINT)
     if [ $res -ge 200 ] && [ $res -lt 400 ]; then
-        logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $ARM_ENDPOINT with returned status code $res'"
+        logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested $ARM_ENDPOINT with returned status code $res'"
     else 
-        logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test $ARM_ENDPOINT with returned status code $res. This endpoint is required for Kubernetes operations against the Azure API'" 
+        logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test $ARM_ENDPOINT with returned status code $res. This endpoint is required for Kubernetes operations against the Azure API'" 
     fi
 else
-    logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to retrieve access token with returned status code $result. Can't check access to $ARM_ENDPOINT'" 
+    logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to retrieve access token with returned status code $result. Can't check access to $ARM_ENDPOINT'" 
 fi
 
 # check access to apiserver
-if [ -z "$APISERVER_FQDN" ]; then
-    logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - WARNING: No apiserver FQDN provided. Skipping apiserver check.'"
+if ! [ -f "${AKS_KUBECONFIG_PATH}" ]; then
+    logs_to_events "AKS.testingTraffic.warning" "echo '$(date) - WARNING: Kubeconfig file not found. Skipping apiserver check.'"
 else
+    APISERVER_FQDN=$(grep server $AKS_KUBECONFIG_PATH | awk -F"server: https://" '{print $2}' | cut -d : -f 1)
+    APISERVER_ENDPOINT="https://${APISERVER_FQDN}/healthz"
     nslookup $APISERVER_FQDN > /dev/null
     if [ $? -eq 0 ]; then
-        logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $APISERVER_FQDN'"
+        logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested DNS resolution to $APISERVER_FQDN'"
         res=$(curl -m $MAX_TIME -s -o /dev/null -w "%{http_code}" --cacert $AKS_CA_CERT_PATH --cert $AKS_CERT_PATH --key $AKS_KEY_PATH $APISERVER_ENDPOINT)
         if [ $res -ge 200 ] && [ $res -lt 400 ]; then
-            logs_to_events "AKS.CSE.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested apiserver $APISERVER_FQDN with returned status code $res'"
+            logs_to_events "AKS.testingTraffic.success" "echo '$(date) - SUCCESS: Successfully tested apiserver $APISERVER_FQDN with returned status code $res'"
         else 
-            logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test $APISERVER_FQDN with returned status code $res. Node might not be able to connect to the apiserver'" 
+            logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test $APISERVER_FQDN with returned status code $res. Node might not be able to connect to the apiserver'" 
         fi
     else
-        logs_to_events "AKS.CSE.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $APISERVER_FQDN. Node might not be able to connect to the apiserver'"
+        logs_to_events "AKS.testingTraffic.failure" "echo '$(date) - ERROR: Failed to test DNS resolution to $APISERVER_FQDN. Node might not be able to connect to the apiserver'"
         dns_trace $APISERVER_FQDN
     fi
 fi
@@ -198,4 +200,6 @@ if [ ! -z "$CUSTOM_ENDPOINT" ]; then
     do
         check_and_curl $url ""
     done
-fi
+fi
+
+logs_to_events "AKS.testingTraffic.end" "echo '$(date) - INFO: Network connectivity check completed'"

parts/linux/cloud-init/artifacts/aks-log-collector.sh

@@ -63,7 +63,6 @@ command -v find >/dev/null && find /dev /etc /var/lib/waagent /var/log -ls >coll
 
 # Collect all installed packages for Ubuntu and Azure Linux
 command -v dpkg >/dev/null && dpkg -l >collect/dpkg.txt 2>&1
-command -v tdnf >/dev/null && tdnf list installed >collect/tdnf.txt 2>&1
 
 # Collect system information
 command -v blkid >/dev/null && blkid >>collect/diskinfo.txt 2>&1

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -379,4 +379,41 @@ should_skip_nvidia_drivers() {
     should_skip=$(echo "$body" | jq -e '.compute.tagsList | map(select(.name | test("SkipGpuDriverInstall"; "i")))[0].value // "false" | test("true"; "i")')
     echo "$should_skip"
 }
+
+start_watch () {
+  capture_time=$(date +%s)
+  start_timestamp=$(date +%H:%M:%S)
+}
+
+stop_watch () {
+
+  local current_time=$(date +%s)
+  local end_timestamp=$(date +%H:%M:%S)
+  local difference_in_seconds=$((current_time - ${1}))
+
+  local elapsed_hours=$(($difference_in_seconds/3600))
+  local elapsed_minutes=$((($difference_in_seconds%3600)/60))
+  local elapsed_seconds=$(($difference_in_seconds%60))
+
+  printf -v benchmark "'${2}' - Total Time Elapsed: %02d:%02d:%02d" $elapsed_hours $elapsed_minutes $elapsed_seconds
+  if [ ${3} == true ]; then
+    printf -v start "     Start time: $script_start_timestamp"
+  else
+    printf -v start "     Start time: $start_timestamp"
+  fi
+  printf -v end "     End Time: $end_timestamp"
+  echo -e "\n$benchmark\n"
+  benchmarks+=("$benchmark")
+  benchmarks+=("$start")
+  benchmarks+=("$end")
+}
+
+show_benchmarks () {
+  echo -e "\nBenchmarks:\n"
+  for i in "${benchmarks[@]}"; do
+    echo "   $i"
+  done
+  echo
+}
+
 #HELPERSEOF

parts/linux/cloud-init/artifacts/ensure-no-dup.sh

@@ -35,8 +35,11 @@ if [[ -z "${bridgeIP}" ]]; then
     exit 1
 fi
 
-podSubnetAddr=$(cat /etc/cni/net.d/10-containerd-net.conflist  | jq -r ".plugins[] | select(.type == \"bridge\") | .ipam.subnet")
-if [[ -z "${podSubnetAddr}" ]]; then
+# cloud-controller-manager assigns the node pod CIDR, then kubelet/containerd put it in the conflist.
+# Parse the conflist to retrieve the pod CIDR (IPv4 is always the first item in `ranges`).
+# If the field we expect isn't there, jq returns "null", so treat that as a failure.
+podSubnetAddr=$(cat /etc/cni/net.d/10-containerd-net.conflist  | jq -r ".plugins[] | select(.type == \"bridge\") | .ipam.ranges[0][0].subnet")
+if [[ -z "${podSubnetAddr}" || "${podSubnetAddr}" == 'null' ]]; then
     echo "could not determine this node's pod ipam subnet range from 10-containerd-net.conflist...exiting early"
     exit 1
 fi
@@ -53,4 +56,4 @@ echo "outputting newly added AKS-DEDUP-PROMISC rules:"
 ebtables -t filter -L OUTPUT 2>/dev/null
 ebtables -t filter -L AKS-DEDUP-PROMISC 2>/dev/null
 exit 0
-#EOF
+#EOF

parts/windows/csecmd.ps1

@@ -13,10 +13,10 @@ $arguments = '
 -AADClientSecret ''{{ GetParameter "encodedServicePrincipalClientSecret" }}''
 -NetworkAPIVersion 2018-08-01
 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log
--CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log';
+-CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete';
 $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin';
 $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1';
 if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' };
-Copy-Item $inputFile $outputFile;
+Copy-Item $inputFile $outputFile -Force;
 Invoke-Expression('{0} {1}' -f $outputFile, $arguments);
 \"

parts/windows/kuberneteswindowssetup.ps1

@@ -210,7 +210,7 @@ $global:RebootNeeded = $false
 
 # Extract cse helper script from ZIP
 [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
-Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\"
+Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -Force
 
 # Dot-source windowscsehelper.ps1 with functions that are called in this script
 . c:\AzureData\windows\windowscsehelper.ps1
@@ -232,7 +232,7 @@ try
     Write-Log "private egress proxy address is '$global:PrivateEgressProxyAddress'"
     # TODO update to use proxy
 
-    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.39.zip"
+    $WindowsCSEScriptsPackage = "aks-windows-cse-scripts-v0.0.40.zip"
     Write-Log "CSEScriptsPackageUrl is $global:CSEScriptsPackageUrl"
     Write-Log "WindowsCSEScriptsPackage is $WindowsCSEScriptsPackage"
     # Old AKS RP sets the full URL (https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-v0.0.11.zip) in CSEScriptsPackageUrl
@@ -248,7 +248,7 @@ try
     Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadAndExpandCSEScriptPackageUrl" -TaskMessage "Start to get CSE scripts. CSEScriptsPackageUrl: $global:CSEScriptsPackageUrl"
     $tempfile = 'c:\csescripts.zip'
     DownloadFileOverHttp -Url $global:CSEScriptsPackageUrl -DestinationPath $tempfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE
-    Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows"
+    Expand-Archive $tempfile -DestinationPath "C:\\AzureData\\windows" -Force
     Remove-Item -Path $tempfile -Force
 
     # Dot-source cse scripts with functions that are called in this script
@@ -488,6 +488,8 @@ finally
     # Generate CSE result so it can be returned as the CSE response in csecmd.ps1
     $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date))
     Write-Log "CSE ExecutionDuration: $ExecutionDuration. ExitCode: $global:ExitCode"
+    # $CSEResultFilePath is used to avoid running CSE multiple times
+    Set-Content -Path $CSEResultFilePath -Value $global:ExitCode -Force
     Logs-To-Event -TaskName "AKS.WindowsCSE.cse_main" -TaskMessage "ExitCode: $global:ExitCode. ErrorMessage: $global:ErrorMessage." 
     # Please not use Write-Log or Logs-To-Events after Stop-Transcript
     Stop-Transcript

parts/windows/windowscsehelper.ps1

@@ -65,6 +65,8 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE=58
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61
+$global:WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG=62
+$global:WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP=63
 
 # Please add new error code for downloading new packages in RP code too
 $global:ErrorCodeNames = @(
@@ -129,7 +131,9 @@ $global:ErrorCodeNames = @(
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE",
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE",
     "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION",
-    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE"
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE",
+    "WINDOWS_CSE_ERROR_UPDATING_KUBE_CLUSTER_CONFIG",
+    "WINDOWS_CSE_ERROR_GET_NODE_IPV6_IP"
 )
 
 # NOTE: KubernetesVersion does not contain "v"