Skip to content

Commit

Permalink
feat: install credential provider support for windows and cache binar…
Browse files Browse the repository at this point in the history 
…y in VHD (#4282)
  • Loading branch information
@mainred
mainred committed Apr 17, 2024
1 parent e46e39c commit 76cba62
Show file tree
Hide file tree
Showing 21 changed files with 1,156 additions and 15 deletions.
7 changes: 6 additions & 1 deletion parts/windows/kuberneteswindowssetup.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,9 @@ $global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}"
$global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}}
$global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}}

# Kubelet credential provider
$global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}"

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("{{GetVariable "windowsEnableCSIProxy" }}");
$global:CsiProxyUrl = "{{GetVariable "windowsCSIProxyURL" }}";
Expand Down Expand Up @@ -292,7 +295,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix {{if IsAKSCustomCloud}}"{{ AKSCustomCloudContainerRegistryDNSSuffix }}"{{else}}""{{end}}

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
44 changes: 44 additions & 0 deletions pkg/agent/baker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1642,6 +1642,50 @@ var _ = Describe("Assert generated customData and cseCmd for Windows", func() {
},
}
}),
Entry("AKSWindows2019 with out of tree credential provider", "AKSWindows2019+ootcredentialprovider", "1.29.0", func(config *datamodel.NodeBootstrappingConfiguration) {
config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
}),
Entry("AKSWindows2019 with custom cloud and out of tree credential provider", "AKSWindows2019+CustomCloud+ootcredentialprovider", "1.29.0",
func(config *datamodel.NodeBootstrappingConfiguration) {
config.ContainerService.Properties.WindowsProfile.AlwaysPullWindowsPauseImage = to.BoolPtr(true)
config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{
Name: "akscustom",
McrURL: "mcr.microsoft.fakecustomcloud",
RepoDepotEndpoint: "https://repodepot.azure.microsoft.fakecustomcloud/ubuntu",
ManagementPortalURL: "https://portal.azure.microsoft.fakecustomcloud/",
PublishSettingsURL: "",
ServiceManagementEndpoint: "https://management.core.microsoft.fakecustomcloud/",
ResourceManagerEndpoint: "https://management.azure.microsoft.fakecustomcloud/",
ActiveDirectoryEndpoint: "https://login.microsoftonline.microsoft.fakecustomcloud/",
GalleryEndpoint: "",
KeyVaultEndpoint: "https://vault.cloudapi.microsoft.fakecustomcloud/",
GraphEndpoint: "https://graph.cloudapi.microsoft.fakecustomcloud/",
ServiceBusEndpoint: "",
BatchManagementEndpoint: "",
StorageEndpointSuffix: "core.microsoft.fakecustomcloud",
SQLDatabaseDNSSuffix: "database.cloudapi.microsoft.fakecustomcloud",
TrafficManagerDNSSuffix: "",
KeyVaultDNSSuffix: "vault.cloudapi.microsoft.fakecustomcloud",
ServiceBusEndpointSuffix: "",
ServiceManagementVMDNSSuffix: "",
ResourceManagerVMDNSSuffix: "cloudapp.azure.microsoft.fakecustomcloud/",
ContainerRegistryDNSSuffix: ".azurecr.microsoft.fakecustomcloud",
CosmosDBDNSSuffix: "documents.core.microsoft.fakecustomcloud/",
TokenAudience: "https://management.core.microsoft.fakecustomcloud/",
ResourceIdentifiers: datamodel.ResourceIdentifiers{
Graph: "",
KeyVault: "",
Datalake: "",
Batch: "",
OperationalInsights: "",
Storage: "",
},
}
config.KubeletConfig["--image-credential-provider-config"] = "c:\\var\\lib\\kubelet\\credential-provider-config.yaml"
config.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
}),
)

})
Expand Down
1 change: 1 addition & 0 deletions pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CSECommand
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''akscustom'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); "
514 changes: 514 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData
View file

Large diffs are not rendered by default.

7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ".azurecr.microsoft.fakecustomcloud"

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S116/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S117/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S118/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("true");
$global:CsiProxyUrl = "https://acs-mirror.azureedge.net/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+K8S119/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
7 changes: 6 additions & 1 deletion pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/b
$global:IsDualStackEnabled = $false
$global:IsAzureCNIOverlayEnabled = $false

# Kubelet credential provider
$global:CredentialProviderURL = ""

# CSI Proxy settings
$global:EnableCsiProxy = [System.Convert]::ToBoolean("false");
$global:CsiProxyUrl = "";
Expand Down Expand Up @@ -288,7 +291,9 @@ try
Get-LogCollectionScripts

Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp


Install-CredentialProvider -KubeDir $global:KubeDir -CustomCloudContainerRegistryDNSSuffix ""

Get-KubePackage -KubeBinariesSASURL $global:KubeBinariesPackageSASURL

$cniBinPath = $global:AzureCNIBinDir
Expand Down
1 change: 1 addition & 0 deletions pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CSECommand
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
powershell.exe -ExecutionPolicy Unrestricted -command " $arguments = ' -MasterIP ''uttestdom-dns-5d7c849e.hcp.southcentralus.azmk8s.io'' -KubeDnsServiceIp ''10.0.0.10'' -MasterFQDNPrefix ''uttestdom'' -Location ''southcentralus'' -TargetEnvironment ''AzurePublicCloud'' -AgentKey '''' -AADClientId ''ClientID'' -AADClientSecret ''U2VjcmV0'' -NetworkAPIVersion 2018-08-01 -LogFile %SYSTEMDRIVE%\AzureData\CustomDataSetupScript.log -CSEResultFilePath %SYSTEMDRIVE%\AzureData\provision.complete'; $inputFile = '%SYSTEMDRIVE%\AzureData\CustomData.bin'; $outputFile = '%SYSTEMDRIVE%\AzureData\CustomDataSetupScript.ps1'; if (!(Test-Path $inputFile)) { throw 'ExitCode: |49|, Output: |WINDOWS_CSE_ERROR_NO_CUSTOM_DATA_BIN|, Error: |C:\AzureData\CustomData.bin does not exist.|' }; Copy-Item $inputFile $outputFile -Force; Invoke-Expression('{0} {1}' -f $outputFile, $arguments); "
Loading

0 comments on commit 76cba62

Please sign in to comment.