feat: return error details in Windows CSE

Azure · Dec 14, 2023 · 729e83c · 729e83c
1 parent 69fa216
commit 729e83c
Show file tree

Hide file tree

Showing 28 changed files with 272 additions and 142 deletions.
diff --git a/parts/windows/csecmd.ps1 b/parts/windows/csecmd.ps1
@@ -16,7 +16,7 @@ $arguments = '
 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log';
 $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin';
 $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1';
-if (!(Test-Path $inputFile)) { echo 49 | Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; };
+if (!(Test-Path $inputFile)) { echo 'ExitCode: \"49\", Output: \"WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN\", Error: \"%SYSTEMDRIVE%\AzureData\CustomData.bin does not exist.\"'| Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; };
 Copy-Item $inputFile $outputFile;
 Invoke-Expression('{0} {1}' -f $outputFile, $arguments);
-\" >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log)) { exit 50; }; $code=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); exit $code
+\" >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log) { $errorMesage=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); throw $errorMesage; }
diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1
@@ -513,13 +513,13 @@ finally
     $ExecutionDuration=$(New-Timespan -Start $StartTime -End $(Get-Date))
     Write-Log "CSE ExecutionDuration: $ExecutionDuration"
 
-    # Windows CSE does not return any error message so we cannot generate below content as the response
-    # $JsonString = "ExitCode: `"{0}`", Output: `"{1}`", Error: `"{2}`", ExecDuration: `"{3}`"" -f $global:ExitCode, "", $global:ErrorMessage, $ExecutionDuration.TotalSeconds
-    Write-Log "Generate CSE result to $CSEResultFilePath : $global:ExitCode"
-    echo $global:ExitCode | Out-File -FilePath $CSEResultFilePath -Encoding utf8
-
-    # Flush stdout to C:\AzureData\CustomDataSetupScript.log
-    [Console]::Out.Flush()
-
     Upload-GuestVMLogs -ExitCode $global:ExitCode
+
+    if ($global:ExitCode -ne 0) {
+        # $JsonString = "ExitCode: `"{0}`", Output: `"{1}`", Error: `"{2}`""
+        # Max length of the full error message returned by Windows CSE is ~256. We use 240 to be safe.
+        $errorMessageLength = "ExitCode: `"$global:ExitCode`", Output: `"$($global:ErrorCodeNames[$global:ExitCode])`", Error: `"`"".Length
+        $turncatedErrorMessage = $global:ErrorMessage.Substring(0, [Math]::Min(240 - $errorMessageLength, $global:ErrorMessage.Length))
+        echo "ExitCode: `"$global:ExitCode`", Output: `"$($global:ErrorCodeNames[$global:ExitCode])`", Error: `"$turncatedErrorMessage`"" | Out-File -FilePath $CSEResultFilePath -Encoding utf8
+    }
 }
diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1
@@ -2,6 +2,7 @@
 # It is better to define functions in the scripts under staging/cse/windows.
 
 # Define all exit codes in Windows CSE
+$global:WINDOWS_CSE_SUCCESS=0
 $global:WINDOWS_CSE_ERROR_UNKNOWN=1 # For unexpected error caught by the catch block in kuberneteswindowssetup.ps1
 $global:WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY=2
 $global:WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE=3
@@ -64,6 +65,70 @@ $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE=59
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION=60
 $global:WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE=61
 
+$global:ErrorCodeNames = @(
+    "WINDOWS_CSE_SUCCESS",
+    "WINDOWS_CSE_ERROR_UNKNOWN",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_FILE_WITH_RETRY",
+    "WINDOWS_CSE_ERROR_INVOKE_EXECUTABLE",
+    "WINDOWS_CSE_ERROR_FILE_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_CHECK_API_SERVER_CONNECTIVITY",
+    "WINDOWS_CSE_ERROR_PAUSE_IMAGE_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_GET_SUBNET_PREFIX",
+    "WINDOWS_CSE_ERROR_GENERATE_TOKEN_FOR_ARM",
+    "WINDOWS_CSE_ERROR_NETWORK_INTERFACES_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_NETWORK_ADAPTER_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_MANAGEMENT_IP_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_CALICO_SERVICE_ACCOUNT_NOT_EXIST",
+    "WINDOWS_CSE_ERROR_CONTAINERD_NOT_INSTALLED",
+    "WINDOWS_CSE_ERROR_CONTAINERD_NOT_RUNNING",
+    "WINDOWS_CSE_ERROR_OPENSSH_NOT_INSTALLED",
+    "WINDOWS_CSE_ERROR_OPENSSH_FIREWALL_NOT_CONFIGURED",
+    "WINDOWS_CSE_ERROR_INVALID_PARAMETER_IN_AZURE_CONFIG",
+    "WINDOWS_CSE_ERROR_NO_DOCKER_TO_BUILD_PAUSE_CONTAINER",
+    "WINDOWS_CSE_ERROR_GET_CA_CERTIFICATES",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CA_CERTIFICATES",
+    "WINDOWS_CSE_ERROR_EMPTY_CA_CERTIFICATES",
+    "WINDOWS_CSE_ERROR_ENABLE_SECURE_TLS",
+    "WINDOWS_CSE_ERROR_GMSA_EXPAND_ARCHIVE",
+    "WINDOWS_CSE_ERROR_GMSA_ENABLE_POWERSHELL_PRIVILEGE",
+    "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_PERMISSION",
+    "WINDOWS_CSE_ERROR_GMSA_SET_REGISTRY_VALUES",
+    "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGEVENTS",
+    "WINDOWS_CSE_ERROR_GMSA_IMPORT_CCGAKVPPLUGINEVENTS",
+    "WINDOWS_CSE_ERROR_NOT_FOUND_MANAGEMENT_IP",
+    "WINDOWS_CSE_ERROR_NOT_FOUND_BUILD_NUMBER",
+    "WINDOWS_CSE_ERROR_NOT_FOUND_PROVISIONING_SCRIPTS",
+    "WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CSE_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CNI_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_HNS_MODULE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CALICO_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_GMSA_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CSI_PROXY_PACKAGE",
+    "WINDOWS_CSE_ERROR_DOWNLOAD_CONTAINERD_PACKAGE",
+    "WINDOWS_CSE_ERROR_SET_TCP_DYNAMIC_PORT_RANGE",
+    "WINDOWS_CSE_ERROR_BUILD_DOCKER_PAUSE_CONTAINER",
+    "WINDOWS_CSE_ERROR_PULL_PAUSE_IMAGE",
+    "WINDOWS_CSE_ERROR_BUILD_TAG_PAUSE_IMAGE",
+    "WINDOWS_CSE_ERROR_CONTAINERD_BINARY_EXIST",
+    "WINDOWS_CSE_ERROR_SET_TCP_EXCLUDE_PORT_RANGE",
+    "WINDOWS_CSE_ERROR_SET_UDP_DYNAMIC_PORT_RANGE",
+    "WINDOWS_CSE_ERROR_SET_UDP_EXCLUDE_PORT_RANGE",
+    "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN",
+    "WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG",
+    "WINDOWS_CSE_ERROR_COPY_LOG_COLLECTION_SCRIPTS",
+    "WINDOWS_CSE_ERROR_RESIZE_OS_DRIVE",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_FAILED",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_TIMEOUT",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_VM_SIZE_NOT_SUPPORTED",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_SET",
+    "WINDOWS_CSE_ERROR_GPU_SKU_INFO_NOT_FOUND",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_DOWNLOAD_FAILURE",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INVALID_SIGNATURE",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_EXCEPTION",
+    "WINDOWS_CSE_ERROR_GPU_DRIVER_INSTALLATION_URL_NOT_EXE"
+)
 
 # NOTE: KubernetesVersion does not contain "v"
 $global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version
@@ -152,7 +217,7 @@ function Set-ExitCode
     )
     Write-Log "Set ExitCode to $ExitCode and exit. Error: $ErrorMessage"
     $global:ExitCode=$ExitCode
-    $global:ErrorMessage=$ErrorMessage
+    $global:ErrorMessage=($ErrorMessage -replace '\r*\n', '')
     exit $ExitCode
 }
 

diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand
@@ -1 +1 @@
-powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus''  -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { echo 49 | Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; }; Copy-Item $inputFile $outputFile; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log)) { exit 50; }; $code=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); exit $code
+powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus''  -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { echo 'ExitCode: "49", Output: "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", Error: "%SYSTEMDRIVE%\AzureData\CustomData.bin does not exist."'| Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; }; Copy-Item $inputFile $outputFile; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log) { $errorMesage=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); throw $errorMesage; }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { echo 49 \| Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; }; Copy-Item $inputFile $outputFile; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log)) { exit 50; }; $code=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); exit $code
		powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\CSEResult.log'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { echo 'ExitCode: "49", Output: "WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN", Error: "%SYSTEMDRIVE%\AzureData\CustomData.bin does not exist."'\| Out-File -FilePath '%SYSTEMDRIVE%\AzureData\CSEResult.log' -Encoding utf8; exit; }; Copy-Item $inputFile $outputFile; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); " >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (Test-Path %SYSTEMDRIVE%\AzureData\CSEResult.log) { $errorMesage=(Get-Content %SYSTEMDRIVE%\AzureData\CSEResult.log); throw $errorMesage; }