diff --git a/e2e/exec.go b/e2e/exec.go index 1c7010b4e18..fa463028bf1 100644 --- a/e2e/exec.go +++ b/e2e/exec.go @@ -153,7 +153,7 @@ func execScriptOnVm(ctx context.Context, s *Scenario, vmPrivateIP, jumpboxPodNam joinedSteps := strings.Join(steps, " && ") - s.T.Log(fmt.Sprintf("Executing script %s:\n---START-SCRIPT---\n%s\n---END-SCRIPT---\n", scriptFileName, script)) + s.T.Log(fmt.Sprintf("Executing script %[1]s using %[2]s:\n---START-SCRIPT---\n%[3]s\n---END-SCRIPT---\n", scriptFileName, interpreter, script.script)) kube := s.Runtime.Cluster.Kube execResult, err := execOnPrivilegedPod(ctx, kube, defaultNamespace, jumpboxPodName, joinedSteps) diff --git a/e2e/node_config.go b/e2e/node_config.go index 0a350d2cfac..191b09eeb75 100644 --- a/e2e/node_config.go +++ b/e2e/node_config.go @@ -24,7 +24,8 @@ import ( ) func getBaseNBC(t *testing.T, cluster *Cluster, vhd *config.Image) *datamodel.NodeBootstrappingConfiguration { - nbc := baseTemplateLinux(t, config.Config.Location, *cluster.Model.Properties.CurrentKubernetesVersion, vhd.Arch) + var nbc *datamodel.NodeBootstrappingConfiguration + if vhd.Distro.IsWindowsDistro() { nbc = baseTemplateWindows(t, config.Config.Location) cert := cluster.Kube.clientCertificate() @@ -35,7 +36,10 @@ func getBaseNBC(t *testing.T, cluster *Cluster, vhd *config.Image) *datamodel.No nbc.SubscriptionID = config.Config.SubscriptionID nbc.ResourceGroupName = *cluster.Model.Properties.NodeResourceGroup nbc.TenantID = *cluster.Model.Identity.TenantID + } else { + nbc = baseTemplateLinux(t, config.Config.Location, *cluster.Model.Properties.CurrentKubernetesVersion, vhd.Arch) } + nbc.ContainerService.Properties.CertificateProfile.CaCertificate = string(cluster.ClusterParams.CACert) nbc.KubeletClientTLSBootstrapToken = &cluster.ClusterParams.BootstrapToken @@ -464,6 +468,7 @@ func baseTemplateWindows(t *testing.T, location string) *datamodel.NodeBootstrap ResourceGroupName: "resourcegroup", ContainerService: &datamodel.ContainerService{ + Location: location, Properties: &datamodel.Properties{ HostedMasterProfile: &datamodel.HostedMasterProfile{}, CertificateProfile: &datamodel.CertificateProfile{}, diff --git a/e2e/vmss.go b/e2e/vmss.go index c66024dccf5..5cc1a244194 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -78,6 +78,7 @@ func createVMSS(ctx context.Context, s *Scenario) *armcompute.VirtualMachineScal }) vmssResp, err := operation.PollUntilDone(ctx, config.DefaultPollUntilDoneOptions) + // fail test, but continue to extract debug information require.NoError(s.T, err, "create vmss %q, check %s for vm logs", s.Runtime.VMSSName, testDir(s.T)) return &vmssResp.VirtualMachineScaleSet @@ -196,7 +197,7 @@ func extractLogsFromVMWindows(ctx context.Context, s *Scenario) { client := config.Azure.VMSSVMRunCommands // Invoke the RunCommand on the VMSS instance - s.T.Log("uploading windows logs to blob storage, may take a few minutes") + s.T.Logf("uploading windows logs to blob storage at %s, may take a few minutes", blobUrl) pollerResp, err := client.BeginCreateOrUpdate( ctx, diff --git a/parts/windows/csecmd.ps1 b/parts/windows/csecmd.ps1 index 6e12a2341f3..4827e23c57a 100644 --- a/parts/windows/csecmd.ps1 +++ b/parts/windows/csecmd.ps1 @@ -1,21 +1,14 @@ powershell.exe -ExecutionPolicy Unrestricted -command \" $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; -if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; +if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; -PowerShell -File $outputFile --MasterIP ''{{ GetKubernetesEndpoint }}'' --KubeDnsServiceIp ''{{ GetParameter "kubeDNSServiceIP" }}'' --MasterFQDNPrefix ''{{ GetParameter "masterEndpointDNSNamePrefix" }}'' --Location ''{{ GetVariable "location" }}'' -{{if UserAssignedIDEnabled}} --UserAssignedClientID ''{{ GetVariable "userAssignedIdentityID" }}'' -{{ end }} --TargetEnvironment ''{{ GetTargetEnvironment }}'' +PowerShell +-File $outputFile -AgentKey ''{{ GetParameter "clientPrivateKey" }}'' --AADClientId ''{{ GetParameter "servicePrincipalClientId" }}'' -AADClientSecret ''{{ GetParameter "encodedServicePrincipalClientSecret" }}'' --NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; -if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; +if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; +$result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); +if ($result -ne '0') { throw $result; }; \" \ No newline at end of file diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 96196368cf3..71236f32da2 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -19,53 +19,38 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "{{ GetKubernetesEndpoint }}" +$KubeDnsServiceIp="{{ GetParameter "kubeDNSServiceIP" }}" +$MasterFQDNPrefix="{{ GetParameter "masterEndpointDNSNamePrefix" }}" +$Location="{{ GetVariable "location" }}" +{{if UserAssignedIDEnabled}} +$UserAssignedClientID="{{ GetVariable "userAssignedIdentityID" }}" +{{ end }} +$TargetEnvironment="{{ GetTargetEnvironment }}" +$AgentKey="{{ GetParameter "clientPrivateKey" }}" +$AADClientId="{{ GetParameter "servicePrincipalClientId" }}" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index ecbd079ca0d..13848af753b 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -30,17 +30,24 @@ func InitializeTemplateGenerator() *TemplateGenerator { // GetNodeBootstrappingPayload get node bootstrapping data. // This function only can be called after the validation of the input NodeBootstrappingConfiguration. func (t *TemplateGenerator) getNodeBootstrappingPayload(config *datamodel.NodeBootstrappingConfiguration) string { - var customData string if config.AgentPoolProfile.IsWindows() { - customData = getCustomDataFromJSON(t.getWindowsNodeCustomDataJSONObject(config)) - } else { - customData = getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + return t.getWindowsNodeBootstrappingPayload(config) } - if config.AgentPoolProfile.IsWindows() { - return base64.StdEncoding.EncodeToString([]byte(customData)) - } + return t.getLinuxNodeBootstrappingPayload(config) +} +func (t *TemplateGenerator) getWindowsNodeBootstrappingPayload(config *datamodel.NodeBootstrappingConfiguration) string { + // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation + // lots of tests fail. + customData := getCustomDataFromJSON(t.getWindowsNodeCustomDataJSONObject(config)) + return base64.StdEncoding.EncodeToString([]byte(customData)) +} + +func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.NodeBootstrappingConfiguration) string { + // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation + // lots of tests fail. + customData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) return getBase64EncodedGzippedCustomScriptFromStr(customData) } @@ -76,7 +83,6 @@ func (t *TemplateGenerator) getWindowsNodeCustomDataJSONObject(config *datamodel } preprovisionCmd := "" - if profile.PreprovisionExtension != nil { preprovisionCmd = makeAgentExtensionScriptCommands(cs, profile) } @@ -134,7 +140,7 @@ func (t *TemplateGenerator) getWindowsNodeCSECommand(config *datamodel.NodeBoots if e != nil { panic(e) } - /* NOTE(qinahao): windows cse cmd uses esapced \" to quote Powershell command in + /* NOTE(qinahao): windows cse cmd uses escaped \" to quote Powershell command in [csecmd.p1](https://github.com/Azure/AgentBaker/blob/master/parts/windows/csecmd.ps1). */ // to not break go template parsing. We switch \" back to " otherwise Azure ARM template will escape \ to be \\\" str = strings.ReplaceAll(str, `\"`, `"`) diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 16b544da58c..c65b1c9b68b 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1052,6 +1052,8 @@ var _ = Describe("Assert generated customData and cseCmd", func() { } }, func(o *nodeBootstrappingOutput) { + Expect(o).ShouldNot(BeNil()) + Expect(o.files["/opt/azure/containers/provision.sh"]).ShouldNot(BeNil()) Expect(o.files["/opt/azure/containers/provision.sh"].encoding).To(Equal(cseVariableEncodingGzip)) cseMain := o.files["/opt/azure/containers/provision.sh"].value httpProxyStr := "export http_proxy=\"http://myproxy.server.com:8080/\"" diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand index 25fd84b4083..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData index f1701e7e66a..0a434a41cf3 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="akscustom" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand index 25fd84b4083..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData index b4a681543d0..b2be3da4cc5 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="akscustom" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CSECommand b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData index 6514c3f247b..b15b64c7b60 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CSECommand b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData index 8f747000cac..6b9b95becf4 100644 --- a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S116/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S116/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S116/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData index 85beabfb6a7..c93f6adad9f 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S117/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S117/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S117/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData index 69768afbf5a..4f351317024 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S118/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S118/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S118/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData index 594d1a45f60..107275a6412 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData index 023b534df8b..1f6dae07da1 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData index 7bdbd44f77c..a8e488d9f19 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CSECommand b/pkg/agent/testdata/AKSWindows2019+K8S119/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+K8S119/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData index bdc506f88d5..5a61bb36703 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CSECommand b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData index c2b41040729..4d7de109845 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CSECommand b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData index 0258d6dabc3..18ac1471c95 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CSECommand b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CSECommand index f2fb2bc1e40..26ebd7ff665 100644 --- a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -UserAssignedClientID ''userAssignedID'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''msi'' -AADClientSecret '''' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret '''' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData index 7991bb55786..0e37d4f045e 100644 --- a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData @@ -19,53 +19,38 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$UserAssignedClientID="userAssignedID" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="msi" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CSECommand b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData index 01423f4542b..658b09b4873 100644 --- a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand index b0c3a605a4d..56c11de8fe0 100644 --- a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand +++ b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand @@ -1 +1 @@ -powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if($result -ne '0') { throw $result; }; " \ No newline at end of file +powershell.exe -ExecutionPolicy Unrestricted -command " $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |$inputFile does not exist.|' }; Copy-Item $inputFile $outputFile -Force; PowerShell -File $outputFile -AgentKey '''' -AADClientSecret ''U2VjcmV0'' -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete >> %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log 2>&1; if (!(Test-Path %SYSTEMDRIVE%\AzureData\provision.complete)) { throw 'ExitCode: |50|, Output: |WINDOWS_CSE_ERROR_NO_CSE_RESULT_LOG|, Error: |C:\AzureData\provision.complete is not generated.|'; }; $result=(Get-Content %SYSTEMDRIVE%\AzureData\provision.complete); if ($result -ne '0') { throw $result; }; " \ No newline at end of file diff --git a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData index 1fad931880b..d4e3049600e 100644 --- a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData @@ -19,53 +19,36 @@ #> [CmdletBinding(DefaultParameterSetName="Standard")] param( - [string] - [ValidateNotNullOrEmpty()] - $MasterIP, - - [parameter()] - [ValidateNotNullOrEmpty()] - $KubeDnsServiceIp, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $MasterFQDNPrefix, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $Location, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AgentKey, - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] $AADClientSecret, # base64 - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $NetworkAPIVersion, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $TargetEnvironment, - # C:\AzureData\provision.complete # MUST keep generating this file when CSE is done and do not change the name # - It is used to avoid running CSE multiple times # - Some customers use this file to check if CSE is done [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $CSEResultFilePath, - - [string] - $UserAssignedClientID + $CSEResultFilePath ) + +# In an ideal world, all these values would be passed to this script in parameters. However, we don't live in an ideal world. +# https://learn.microsoft.com/en-gb/troubleshoot/windows-client/shell-experience/command-line-string-limitation + +$MasterIP = "uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io" +$KubeDnsServiceIp="10.0.0.10" +$MasterFQDNPrefix="uttestdom" +$Location="southcentralus" + +$TargetEnvironment="AzurePublicCloud" +$AgentKey="" +$AADClientId="ClientID" +$NetworkAPIVersion="2018-08-01" + # Do not parse the start time from $LogFile to simplify the logic $StartTime=Get-Date $global:ExitCode=0 diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 33d996f5f96..80eedb389f5 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -77,12 +77,22 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa } // getWindowsCustomDataVariables returns custom data for Windows. -/* TODO(qinhao): combine this function with `getCSECommandVariables` after we support passing variables -from cse command to customdata. */ func getWindowsCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { + return getCSECommandVariables(config) +} + +func getCSECommandVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { cs := config.ContainerService - // these variables is subet of. - customData := map[string]interface{}{ + profile := config.AgentPoolProfile + + // this method is called for both windows and linux. If there's no windows profile, then let's just + // use a blank one. + windowsProfile := cs.Properties.WindowsProfile + if windowsProfile == nil { + windowsProfile = &datamodel.WindowsProfile{} + } + + return map[string]interface{}{ "tenantID": config.TenantID, "subscriptionId": config.SubscriptionID, "resourceGroup": config.ResourceGroupName, @@ -91,6 +101,7 @@ func getWindowsCustomDataVariables(config *datamodel.NodeBootstrappingConfigurat "subnetName": cs.Properties.GetSubnetName(), "nsgName": cs.Properties.GetNSGName(), "virtualNetworkName": cs.Properties.GetVirtualNetworkName(), + "virtualNetworkResourceGroupName": cs.Properties.GetVNetResourceGroupName(), "routeTableName": cs.Properties.GetRouteTableName(), "primaryAvailabilitySetName": cs.Properties.GetPrimaryAvailabilitySetName(), "primaryScaleSetName": config.PrimaryScaleSetName, @@ -98,54 +109,28 @@ func getWindowsCustomDataVariables(config *datamodel.NodeBootstrappingConfigurat "useInstanceMetadata": useInstanceMetadata(cs), "loadBalancerSku": cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku, "excludeMasterFromStandardLB": true, - "windowsEnableCSIProxy": cs.Properties.WindowsProfile.IsCSIProxyEnabled(), - "windowsCSIProxyURL": cs.Properties.WindowsProfile.CSIProxyURL, - "windowsProvisioningScriptsPackageURL": cs.Properties.WindowsProfile.ProvisioningScriptsPackageURL, - "windowsPauseImageURL": cs.Properties.WindowsProfile.WindowsPauseImageURL, - "alwaysPullWindowsPauseImage": strconv.FormatBool(cs.Properties.WindowsProfile.IsAlwaysPullWindowsPauseImage()), - "windowsCalicoPackageURL": cs.Properties.WindowsProfile.WindowsCalicoPackageURL, + "maximumLoadBalancerRuleCount": getMaximumLoadBalancerRuleCount(cs), + "userAssignedIdentityID": config.UserAssignedIdentityClientID, + "isVHD": isVHD(profile), + "gpuNode": strconv.FormatBool(config.EnableNvidia), + "sgxNode": strconv.FormatBool(datamodel.IsSgxEnabledSKU(profile.VMSize)), "configGPUDriverIfNeeded": config.ConfigGPUDriverIfNeeded, - "windowsSecureTlsEnabled": cs.Properties.WindowsProfile.IsWindowsSecureTlsEnabled(), - "windowsGmsaPackageUrl": cs.Properties.WindowsProfile.WindowsGmsaPackageUrl, - "windowsGpuDriverURL": cs.Properties.WindowsProfile.GpuDriverURL, - "windowsCSEScriptsPackageURL": cs.Properties.WindowsProfile.CseScriptsPackageURL, + "enableGPUDevicePluginIfNeeded": config.EnableGPUDevicePluginIfNeeded, + "migNode": strconv.FormatBool(datamodel.IsMIGNode(config.GPUInstanceProfile)), + "gpuInstanceProfile": config.GPUInstanceProfile, + "windowsEnableCSIProxy": windowsProfile.IsCSIProxyEnabled(), + "windowsPauseImageURL": windowsProfile.WindowsPauseImageURL, + "windowsCSIProxyURL": windowsProfile.CSIProxyURL, + "windowsProvisioningScriptsPackageURL": windowsProfile.ProvisioningScriptsPackageURL, + "alwaysPullWindowsPauseImage": strconv.FormatBool(windowsProfile.IsAlwaysPullWindowsPauseImage()), + "windowsCalicoPackageURL": windowsProfile.WindowsCalicoPackageURL, + "windowsSecureTlsEnabled": windowsProfile.IsWindowsSecureTlsEnabled(), + "windowsGmsaPackageUrl": windowsProfile.WindowsGmsaPackageUrl, + "windowsGpuDriverURL": windowsProfile.GpuDriverURL, + "windowsCSEScriptsPackageURL": windowsProfile.CseScriptsPackageURL, "isDisableWindowsOutboundNat": strconv.FormatBool(config.AgentPoolProfile.IsDisableWindowsOutboundNat()), "isSkipCleanupNetwork": strconv.FormatBool(config.AgentPoolProfile.IsSkipCleanupNetwork()), } - - return customData -} - -func getCSECommandVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { - cs := config.ContainerService - profile := config.AgentPoolProfile - return map[string]interface{}{ - "tenantID": config.TenantID, - "subscriptionId": config.SubscriptionID, - "resourceGroup": config.ResourceGroupName, - "location": cs.Location, - "vmType": cs.Properties.GetVMType(), - "subnetName": cs.Properties.GetSubnetName(), - "nsgName": cs.Properties.GetNSGName(), - "virtualNetworkName": cs.Properties.GetVirtualNetworkName(), - "virtualNetworkResourceGroupName": cs.Properties.GetVNetResourceGroupName(), - "routeTableName": cs.Properties.GetRouteTableName(), - "primaryAvailabilitySetName": cs.Properties.GetPrimaryAvailabilitySetName(), - "primaryScaleSetName": config.PrimaryScaleSetName, - "useManagedIdentityExtension": useManagedIdentity(cs), - "useInstanceMetadata": useInstanceMetadata(cs), - "loadBalancerSku": cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku, - "excludeMasterFromStandardLB": true, - "maximumLoadBalancerRuleCount": getMaximumLoadBalancerRuleCount(cs), - "userAssignedIdentityID": config.UserAssignedIdentityClientID, - "isVHD": isVHD(profile), - "gpuNode": strconv.FormatBool(config.EnableNvidia), - "sgxNode": strconv.FormatBool(datamodel.IsSgxEnabledSKU(profile.VMSize)), - "configGPUDriverIfNeeded": config.ConfigGPUDriverIfNeeded, - "enableGPUDevicePluginIfNeeded": config.EnableGPUDevicePluginIfNeeded, - "migNode": strconv.FormatBool(datamodel.IsMIGNode(config.GPUInstanceProfile)), - "gpuInstanceProfile": config.GPUInstanceProfile, - } } func useManagedIdentity(cs *datamodel.ContainerService) string { diff --git a/pkg/agent/variables_test.go b/pkg/agent/variables_test.go new file mode 100644 index 00000000000..5675932c6b8 --- /dev/null +++ b/pkg/agent/variables_test.go @@ -0,0 +1,763 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package agent + +import ( + "github.com/Azure/agentbaker/pkg/agent/datamodel" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Windows custom data variables check", func() { + var ( + config *datamodel.NodeBootstrappingConfiguration + ) + + BeforeEach(func() { + config = getDefaultNBC() + }) + + It("sets tenantId", func() { + config.TenantID = "test tenant id" + vars := getWindowsCustomDataVariables(config) + Expect(vars["tenantID"]).To(Equal("test tenant id")) + }) + + It("sets subscriptionId", func() { + config.SubscriptionID = "test sub id" + vars := getWindowsCustomDataVariables(config) + Expect(vars["subscriptionId"]).To(Equal("test sub id")) + }) + + It("sets resourceGroup", func() { + config.ResourceGroupName = "test rg" + vars := getWindowsCustomDataVariables(config) + Expect(vars["resourceGroup"]).To(Equal("test rg")) + }) + + It("sets location", func() { + config.ContainerService.Location = "test loc" + vars := getWindowsCustomDataVariables(config) + Expect(vars["location"]).To(Equal("test loc")) + }) + + It("sets vmType for vmss", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.VirtualMachineScaleSets + vars := getWindowsCustomDataVariables(config) + Expect(vars["vmType"]).To(Equal("vmss")) + }) + + It("sets vmType for vmas", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.AvailabilitySet + vars := getWindowsCustomDataVariables(config) + Expect(vars["vmType"]).To(Equal("standard")) + }) + + It("sets subnetName for custom subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = + "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/aks-vnet-07752737/subnet/testSubnetName" + + vars := getWindowsCustomDataVariables(config) + Expect(vars["subnetName"]).To(Equal("testSubnetName")) + }) + + It("sets subnetName for regular subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = "" + + vars := getWindowsCustomDataVariables(config) + Expect(vars["subnetName"]).To(Equal("aks-subnet")) + }) + + It("sets nsgName", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getWindowsCustomDataVariables(config) + Expect(vars["nsgName"]).To(Equal("aks-agentpool-36873793-nsg")) + }) + + It("sets virtualNetworkName for custom subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = + "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/testVnetName/subnet/testSubnetName" + + vars := getWindowsCustomDataVariables(config) + Expect(vars["virtualNetworkName"]).To(Equal("testVnetName")) + }) + + It("sets virtualNetworkName for regular subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = "" + config.ContainerService.Properties.ClusterID = "36873793" + + vars := getWindowsCustomDataVariables(config) + Expect(vars["virtualNetworkName"]).To(Equal("aks-vnet-36873793")) + }) + + It("sets routeTableName", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getWindowsCustomDataVariables(config) + Expect(vars["routeTableName"]).To(Equal("aks-agentpool-36873793-routetable")) + }) + + It("sets primaryAvailabilitySetName to nothing when no availability set", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getWindowsCustomDataVariables(config) + Expect(vars["primaryAvailabilitySetName"]).To(Equal("")) + }) + + It("sets primaryAvailabilitySetName when there is an availability set", func() { + config.ContainerService.Properties.ClusterID = "36873793" + config.ContainerService.Properties.AgentPoolProfiles[0].Name = "agentpoolname" + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.AvailabilitySet + vars := getWindowsCustomDataVariables(config) + Expect(vars["primaryAvailabilitySetName"]).To(Equal("agentpoolname-availabilitySet-36873793")) + }) + + It("sets primaryScaleSetName", func() { + config.PrimaryScaleSetName = "primary ss name" + vars := getWindowsCustomDataVariables(config) + Expect(vars["primaryScaleSetName"]).To(Equal("primary ss name")) + }) + + It("sets useManagedIdentityExtension to true when using managed identity", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true + vars := getWindowsCustomDataVariables(config) + Expect(vars["useManagedIdentityExtension"]).To(Equal("true")) + }) + + It("sets useManagedIdentityExtension to false when not using managed identity", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = false + vars := getWindowsCustomDataVariables(config) + Expect(vars["useManagedIdentityExtension"]).To(Equal("false")) + }) + + It("sets useInstanceMetadata to true when using instance metadata", func() { + val := true + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = &val + vars := getWindowsCustomDataVariables(config) + Expect(vars["useInstanceMetadata"]).To(Equal("true")) + }) + + It("sets useInstanceMetadata to false when not using instance metadata", func() { + val := false + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = &val + vars := getWindowsCustomDataVariables(config) + Expect(vars["useInstanceMetadata"]).To(Equal("false")) + }) + + It("sets loadBalancerSku ", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = "load balencer sku" + vars := getWindowsCustomDataVariables(config) + Expect(vars["loadBalancerSku"]).To(Equal("load balencer sku")) + }) + + It("sets excludeMasterFromStandardLB", func() { + // at the time of writing this test, this variable was hard coded to true + vars := getWindowsCustomDataVariables(config) + Expect(vars["excludeMasterFromStandardLB"]).To(Equal(true)) + }) + + It("sets windowsEnableCSIProxy to true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(true)) + }) + + It("sets windowsEnableCSIProxy to false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(false)) + }) + + It("sets windowsEnableCSIProxy to the default when no proxy set", func() { + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = nil + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(false)) + }) + + It("sets windowsCSIProxyURL", func() { + config.ContainerService.Properties.WindowsProfile.CSIProxyURL = "csi proxy url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsCSIProxyURL"]).To(Equal("csi proxy url")) + }) + + It("sets windowsProvisioningScriptsPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.ProvisioningScriptsPackageURL = "prov script url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsProvisioningScriptsPackageURL"]).To(Equal("prov script url")) + }) + + It("sets windowsPauseImageURL", func() { + config.ContainerService.Properties.WindowsProfile.WindowsPauseImageURL = "pause image url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsPauseImageURL"]).To(Equal("pause image url")) + }) + + It("sets alwaysPullWindowsPauseImage to true when true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("true")) + }) + + It("sets alwaysPullWindowsPauseImage to false when false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("false")) + }) + + It("sets alwaysPullWindowsPauseImage to false when nil", func() { + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = nil + vars := getWindowsCustomDataVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("false")) + }) + + It("sets windowsCalicoPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.WindowsCalicoPackageURL = "calico package url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsCalicoPackageURL"]).To(Equal("calico package url")) + }) + + It("sets configGPUDriverIfNeeded to true", func() { + config.ConfigGPUDriverIfNeeded = true + vars := getWindowsCustomDataVariables(config) + Expect(vars["configGPUDriverIfNeeded"]).To(Equal(true)) + }) + + It("sets configGPUDriverIfNeeded to false", func() { + config.ConfigGPUDriverIfNeeded = false + vars := getWindowsCustomDataVariables(config) + Expect(vars["configGPUDriverIfNeeded"]).To(Equal(false)) + }) + + It("sets windowsSecureTlsEnabled to true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(true)) + }) + + It("sets windowsSecureTlsEnabled to false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(false)) + }) + + It("sets windowsSecureTlsEnabled to false when nil", func() { + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = nil + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(false)) + }) + + It("sets windowsGmsaPackageUrl", func() { + config.ContainerService.Properties.WindowsProfile.WindowsGmsaPackageUrl = "gsma package url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsGmsaPackageUrl"]).To(Equal("gsma package url")) + }) + + It("sets windowsGpuDriverURL", func() { + config.ContainerService.Properties.WindowsProfile.GpuDriverURL = "gpu driver url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsGpuDriverURL"]).To(Equal("gpu driver url")) + }) + + It("sets windowsCSEScriptsPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL = "cse scripts url" + vars := getWindowsCustomDataVariables(config) + Expect(vars["windowsCSEScriptsPackageURL"]).To(Equal("cse scripts url")) + }) + + It("sets isDisableWindowsOutboundNat to true", func() { + value := true + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: &value, + } + vars := getWindowsCustomDataVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("true")) + }) + + It("sets isDisableWindowsOutboundNat to false", func() { + value := false + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: &value, + } + vars := getWindowsCustomDataVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("false")) + }) + + It("sets isDisableWindowsOutboundNat to false when nil", func() { + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: nil, + } + vars := getWindowsCustomDataVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("false")) + }) + + It("sets isSkipCleanupNetwork to true", func() { + value := true + config.AgentPoolProfile.NotRebootWindowsNode = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("true")) + }) + + It("sets isSkipCleanupNetwork to false", func() { + value := false + config.AgentPoolProfile.NotRebootWindowsNode = &value + vars := getWindowsCustomDataVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("false")) + }) + It("sets isSkipCleanupNetwork to false when nil", func() { + config.AgentPoolProfile.NotRebootWindowsNode = nil + vars := getWindowsCustomDataVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("false")) + }) + +}) + +var _ = Describe("Windows CSE variables check", func() { + var ( + config *datamodel.NodeBootstrappingConfiguration + ) + + BeforeEach(func() { + config = getDefaultNBC() + }) + + It("sets maximumLoadBalancerRuleCount", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.MaximumLoadBalancerRuleCount = 5 + vars := getCSECommandVariables(config) + Expect(vars["maximumLoadBalancerRuleCount"]).To(Equal(5)) + }) + + It("sets userAssignedIdentityID", func() { + config.UserAssignedIdentityClientID = "the identity id" + vars := getCSECommandVariables(config) + Expect(vars["userAssignedIdentityID"]).To(Equal("the identity id")) + }) + + It("sets tenantId", func() { + config.TenantID = "test tenant id" + vars := getCSECommandVariables(config) + Expect(vars["tenantID"]).To(Equal("test tenant id")) + }) + + It("sets subscriptionId", func() { + config.SubscriptionID = "test sub id" + vars := getCSECommandVariables(config) + Expect(vars["subscriptionId"]).To(Equal("test sub id")) + }) + + It("sets resourceGroup", func() { + config.ResourceGroupName = "test rg" + vars := getCSECommandVariables(config) + Expect(vars["resourceGroup"]).To(Equal("test rg")) + }) + + It("sets location", func() { + config.ContainerService.Location = "test loc" + vars := getCSECommandVariables(config) + Expect(vars["location"]).To(Equal("test loc")) + }) + + It("sets vmType for vmss", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.VirtualMachineScaleSets + vars := getCSECommandVariables(config) + Expect(vars["vmType"]).To(Equal("vmss")) + }) + + It("sets vmType for vmas", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.AvailabilitySet + vars := getCSECommandVariables(config) + Expect(vars["vmType"]).To(Equal("standard")) + }) + + It("sets subnetName for custom subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = + "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/aks-vnet-07752737/subnet/testSubnetName" + + vars := getCSECommandVariables(config) + Expect(vars["subnetName"]).To(Equal("testSubnetName")) + }) + + It("sets subnetName for regular subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = "" + + vars := getCSECommandVariables(config) + Expect(vars["subnetName"]).To(Equal("aks-subnet")) + }) + + It("sets nsgName", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getCSECommandVariables(config) + Expect(vars["nsgName"]).To(Equal("aks-agentpool-36873793-nsg")) + }) + + It("sets virtualNetworkName for custom subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = + "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/testVnetName/subnet/testSubnetName" + + vars := getCSECommandVariables(config) + Expect(vars["virtualNetworkName"]).To(Equal("testVnetName")) + }) + + It("sets virtualNetworkName for regular subnet", func() { + config.ContainerService.Properties.AgentPoolProfiles[0].VnetSubnetID = "" + config.ContainerService.Properties.ClusterID = "36873793" + + vars := getCSECommandVariables(config) + Expect(vars["virtualNetworkName"]).To(Equal("aks-vnet-36873793")) + }) + + It("sets routeTableName", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getCSECommandVariables(config) + Expect(vars["routeTableName"]).To(Equal("aks-agentpool-36873793-routetable")) + }) + + It("sets primaryAvailabilitySetName to nothing when no availability set", func() { + config.ContainerService.Properties.ClusterID = "36873793" + vars := getCSECommandVariables(config) + Expect(vars["primaryAvailabilitySetName"]).To(Equal("")) + }) + + It("sets primaryAvailabilitySetName when there is an availability set", func() { + config.ContainerService.Properties.ClusterID = "36873793" + config.ContainerService.Properties.AgentPoolProfiles[0].Name = "agentpoolname" + config.ContainerService.Properties.AgentPoolProfiles[0].AvailabilityProfile = datamodel.AvailabilitySet + vars := getCSECommandVariables(config) + Expect(vars["primaryAvailabilitySetName"]).To(Equal("agentpoolname-availabilitySet-36873793")) + }) + + It("sets primaryScaleSetName", func() { + config.PrimaryScaleSetName = "primary ss name" + vars := getCSECommandVariables(config) + Expect(vars["primaryScaleSetName"]).To(Equal("primary ss name")) + }) + + It("sets useManagedIdentityExtension to true when using managed identity", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true + vars := getCSECommandVariables(config) + Expect(vars["useManagedIdentityExtension"]).To(Equal("true")) + }) + + It("sets useManagedIdentityExtension to false when not using managed identity", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = false + vars := getCSECommandVariables(config) + Expect(vars["useManagedIdentityExtension"]).To(Equal("false")) + }) + + It("sets useInstanceMetadata to true when using instance metadata", func() { + val := true + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = &val + vars := getCSECommandVariables(config) + Expect(vars["useInstanceMetadata"]).To(Equal("true")) + }) + + It("sets useInstanceMetadata to false when not using instance metadata", func() { + val := false + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = &val + vars := getCSECommandVariables(config) + Expect(vars["useInstanceMetadata"]).To(Equal("false")) + }) + + It("sets loadBalancerSku ", func() { + config.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = "load balencer sku" + vars := getCSECommandVariables(config) + Expect(vars["loadBalancerSku"]).To(Equal("load balencer sku")) + }) + + It("sets excludeMasterFromStandardLB", func() { + // at the time of writing this test, this variable was hard coded to true + vars := getCSECommandVariables(config) + Expect(vars["excludeMasterFromStandardLB"]).To(Equal(true)) + }) + + It("sets windowsEnableCSIProxy to true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = &value + vars := getCSECommandVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(true)) + }) + + It("sets windowsEnableCSIProxy to false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = &value + vars := getCSECommandVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(false)) + }) + + It("sets windowsEnableCSIProxy to the default when no proxy set", func() { + config.ContainerService.Properties.WindowsProfile.EnableCSIProxy = nil + vars := getCSECommandVariables(config) + Expect(vars["windowsEnableCSIProxy"]).To(Equal(false)) + }) + + It("sets windowsCSIProxyURL", func() { + config.ContainerService.Properties.WindowsProfile.CSIProxyURL = "csi proxy url" + vars := getCSECommandVariables(config) + Expect(vars["windowsCSIProxyURL"]).To(Equal("csi proxy url")) + }) + + It("sets windowsProvisioningScriptsPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.ProvisioningScriptsPackageURL = "prov script url" + vars := getCSECommandVariables(config) + Expect(vars["windowsProvisioningScriptsPackageURL"]).To(Equal("prov script url")) + }) + + It("sets windowsPauseImageURL", func() { + config.ContainerService.Properties.WindowsProfile.WindowsPauseImageURL = "pause image url" + vars := getCSECommandVariables(config) + Expect(vars["windowsPauseImageURL"]).To(Equal("pause image url")) + }) + + It("sets alwaysPullWindowsPauseImage to true when true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = &value + vars := getCSECommandVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("true")) + }) + + It("sets alwaysPullWindowsPauseImage to false when false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = &value + vars := getCSECommandVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("false")) + }) + + It("sets alwaysPullWindowsPauseImage to false when nil", func() { + config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = nil + vars := getCSECommandVariables(config) + Expect(vars["alwaysPullWindowsPauseImage"]).To(Equal("false")) + }) + + It("sets windowsCalicoPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.WindowsCalicoPackageURL = "calico package url" + vars := getCSECommandVariables(config) + Expect(vars["windowsCalicoPackageURL"]).To(Equal("calico package url")) + }) + + It("sets configGPUDriverIfNeeded to true", func() { + config.ConfigGPUDriverIfNeeded = true + vars := getCSECommandVariables(config) + Expect(vars["configGPUDriverIfNeeded"]).To(Equal(true)) + }) + + It("sets configGPUDriverIfNeeded to false", func() { + config.ConfigGPUDriverIfNeeded = false + vars := getCSECommandVariables(config) + Expect(vars["configGPUDriverIfNeeded"]).To(Equal(false)) + }) + + It("sets windowsSecureTlsEnabled to true", func() { + value := true + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = &value + vars := getCSECommandVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(true)) + }) + + It("sets windowsSecureTlsEnabled to false", func() { + value := false + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = &value + vars := getCSECommandVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(false)) + }) + + It("sets windowsSecureTlsEnabled to false when nil", func() { + config.ContainerService.Properties.WindowsProfile.WindowsSecureTlsEnabled = nil + vars := getCSECommandVariables(config) + Expect(vars["windowsSecureTlsEnabled"]).To(Equal(false)) + }) + + It("sets windowsGmsaPackageUrl", func() { + config.ContainerService.Properties.WindowsProfile.WindowsGmsaPackageUrl = "gsma package url" + vars := getCSECommandVariables(config) + Expect(vars["windowsGmsaPackageUrl"]).To(Equal("gsma package url")) + }) + + It("sets windowsGpuDriverURL", func() { + config.ContainerService.Properties.WindowsProfile.GpuDriverURL = "gpu driver url" + vars := getCSECommandVariables(config) + Expect(vars["windowsGpuDriverURL"]).To(Equal("gpu driver url")) + }) + + It("sets windowsCSEScriptsPackageURL", func() { + config.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL = "cse scripts url" + vars := getCSECommandVariables(config) + Expect(vars["windowsCSEScriptsPackageURL"]).To(Equal("cse scripts url")) + }) + + It("sets isDisableWindowsOutboundNat to true", func() { + value := true + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: &value, + } + vars := getCSECommandVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("true")) + }) + + It("sets isDisableWindowsOutboundNat to false", func() { + value := false + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: &value, + } + vars := getCSECommandVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("false")) + }) + + It("sets isDisableWindowsOutboundNat to false when nil", func() { + config.AgentPoolProfile.AgentPoolWindowsProfile = &datamodel.AgentPoolWindowsProfile{ + DisableOutboundNat: nil, + } + vars := getCSECommandVariables(config) + Expect(vars["isDisableWindowsOutboundNat"]).To(Equal("false")) + }) + + It("sets isSkipCleanupNetwork to true", func() { + value := true + config.AgentPoolProfile.NotRebootWindowsNode = &value + vars := getCSECommandVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("true")) + }) + + It("sets isSkipCleanupNetwork to false", func() { + value := false + config.AgentPoolProfile.NotRebootWindowsNode = &value + vars := getCSECommandVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("false")) + }) + It("sets isSkipCleanupNetwork to false when nil", func() { + config.AgentPoolProfile.NotRebootWindowsNode = nil + vars := getCSECommandVariables(config) + Expect(vars["isSkipCleanupNetwork"]).To(Equal("false")) + }) + +}) + +func getDefaultNBC() *datamodel.NodeBootstrappingConfiguration { + cs := &datamodel.ContainerService{ + Location: "southcentralus", + Type: "Microsoft.ContainerService/ManagedClusters", + Properties: &datamodel.Properties{ + ClusterID: "36873792", + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorType: datamodel.Kubernetes, + OrchestratorVersion: "1.16.15", + KubernetesConfig: &datamodel.KubernetesConfig{}, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + DNSPrefix: "uttestdom", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{ + { + Name: "agent2", + VMSize: "Standard_DS1_v2", + StorageProfile: "ManagedDisks", + OSType: datamodel.Linux, + VnetSubnetID: "/subscriptions/359833f5/resourceGroups/MC_rg/providers/Microsoft.Network/virtualNetworks/aks-vnet-07752737/subnet/subnet1", + AvailabilityProfile: datamodel.VirtualMachineScaleSets, + Distro: datamodel.AKSUbuntu1604, + }, + }, + WindowsProfile: &datamodel.WindowsProfile{}, + LinuxProfile: &datamodel.LinuxProfile{ + AdminUsername: "azureuser", + }, + ServicePrincipalProfile: &datamodel.ServicePrincipalProfile{ + ClientID: "ClientID", + Secret: "Secret", + }, + }, + } + cs.Properties.LinuxProfile.SSH.PublicKeys = []datamodel.PublicKey{{ + KeyData: string("testsshkey"), + }} + + agentPool := cs.Properties.AgentPoolProfiles[0] + + k8sComponents := &datamodel.K8sComponents{} + + kubeletConfig := map[string]string{ + "--address": "0.0.0.0", + "--pod-manifest-path": "/etc/kubernetes/manifests", + "--cloud-provider": "azure", + "--cloud-config": "/etc/kubernetes/azure.json", + "--azure-container-registry-config": "/etc/kubernetes/azure.json", + "--cluster-domain": "cluster.local", + "--cluster-dns": "10.0.0.10", + "--cgroups-per-qos": "true", + "--tls-cert-file": "/etc/kubernetes/certs/kubeletserver.crt", + "--tls-private-key-file": "/etc/kubernetes/certs/kubeletserver.key", + "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll + "--max-pods": "110", + "--node-status-update-frequency": "10s", + "--image-gc-high-threshold": "85", + "--image-gc-low-threshold": "80", + "--event-qps": "0", + "--pod-max-pids": "-1", + "--enforce-node-allocatable": "pods", + "--streaming-connection-idle-timeout": "4h0m0s", + "--rotate-certificates": "true", + "--read-only-port": "10255", + "--protect-kernel-defaults": "true", + "--resolv-conf": "/etc/resolv.conf", + "--anonymous-auth": "false", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--authentication-token-webhook": "true", + "--authorization-mode": "Webhook", + "--eviction-hard": "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%", + "--feature-gates": "RotateKubeletServerCertificate=true,a=b,PodPriority=true,x=y", + "--system-reserved": "cpu=2,memory=1Gi", + "--kube-reserved": "cpu=100m,memory=1638Mi", + } + + galleries := map[string]datamodel.SIGGalleryConfig{ + "AKSUbuntu": { + GalleryName: "aksubuntu", + ResourceGroup: "resourcegroup", + }, + "AKSCBLMariner": { + GalleryName: "akscblmariner", + ResourceGroup: "resourcegroup", + }, + "AKSAzureLinux": { + GalleryName: "aksazurelinux", + ResourceGroup: "resourcegroup", + }, + "AKSWindows": { + GalleryName: "akswindows", + ResourceGroup: "resourcegroup", + }, + "AKSUbuntuEdgeZone": { + GalleryName: "AKSUbuntuEdgeZone", + ResourceGroup: "AKS-Ubuntu-EdgeZone", + }, + } + sigConfig := &datamodel.SIGConfig{ + TenantID: "sometenantid", + SubscriptionID: "somesubid", + Galleries: galleries, + } + + config := &datamodel.NodeBootstrappingConfiguration{ + ContainerService: cs, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: k8sComponents, + AgentPoolProfile: agentPool, + TenantID: "tenantID", + SubscriptionID: "subID", + ResourceGroupName: "resourceGroupName", + UserAssignedIdentityClientID: "userAssignedID", + ConfigGPUDriverIfNeeded: true, + EnableGPUDevicePluginIfNeeded: false, + EnableKubeletConfigFile: false, + EnableNvidia: false, + FIPSEnabled: false, + KubeletConfig: kubeletConfig, + PrimaryScaleSetName: "aks-agent2-36873793-vmss", + SIGConfig: *sigConfig, + } + + return config +}