Merge branch 'master' into jiashun/ss-cve-3

Azure · Aug 26, 2024 · 40e322e · 40e322e
2 parents 026053d + d238370
commit 40e322e
Show file tree

Hide file tree

Showing 177 changed files with 305 additions and 177 deletions.
diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go
@@ -582,6 +582,10 @@ func Test_ubuntu2204(t *testing.T) {
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
 				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-22.04-gen2"
 				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-22.04-gen2"
+				// Check that we don't leak these secrets if they're
+				// set (which they mostly aren't in these scenarios).
+				nbc.ContainerService.Properties.CertificateProfile.ClientPrivateKey = "client cert private key"
+				nbc.ContainerService.Properties.ServicePrincipalProfile.Secret = "SP secret"
 			},
 			LiveVMValidators: []*LiveVMValidator{
 				containerdVersionValidator("1.7.20-1"),

diff --git a/e2e/validation.go b/e2e/validation.go
@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"strings"
 	"testing"
@@ -42,7 +43,7 @@ func runLiveVMValidators(ctx context.Context, t *testing.T, vmssName, privateIP,
 		return fmt.Errorf("while running live validator for node %s, unable to get non host debug pod name: %w", vmssName, err)
 	}
 
-	validators := commonLiveVMValidators()
+	validators := commonLiveVMValidators(opts)
 	if opts.scenario.LiveVMValidators != nil {
 		validators = append(validators, opts.scenario.LiveVMValidators...)
 	}
@@ -74,8 +75,8 @@ func runLiveVMValidators(ctx context.Context, t *testing.T, vmssName, privateIP,
 	return nil
 }
 
-func commonLiveVMValidators() []*LiveVMValidator {
-	return []*LiveVMValidator{
+func commonLiveVMValidators(opts *scenarioRunOpts) []*LiveVMValidator {
+	validators := []*LiveVMValidator{
 		{
 			Description: "assert /etc/default/kubelet should not contain dynamic config dir flag",
 			Command:     "cat /etc/default/kubelet",
@@ -146,4 +147,26 @@ func commonLiveVMValidators() []*LiveVMValidator {
 			IsPodNetwork: true,
 		},
 	}
+	validators = append(validators, leakedSecretsValidators(opts)...)
+	return validators
+}
+
+func leakedSecretsValidators(opts *scenarioRunOpts) []*LiveVMValidator {
+	logPath := "/var/log/azure/cluster-provision.log"
+	clientPrivateKey := opts.nbc.ContainerService.Properties.CertificateProfile.ClientPrivateKey
+	spSecret := opts.nbc.ContainerService.Properties.ServicePrincipalProfile.Secret
+	bootstrapToken := *opts.nbc.KubeletClientTLSBootstrapToken
+
+	b64Encoded := func(val string) string {
+		return base64.StdEncoding.EncodeToString([]byte(val))
+	}
+	return []*LiveVMValidator{
+		// Base64 encoded in baker.go (GetKubeletClientKey)
+		FileExcludesContentsValidator(logPath, b64Encoded(clientPrivateKey), "client private key"),
+		// Base64 encoded in baker.go (GetServicePrincipalSecret)
+		FileExcludesContentsValidator(logPath, b64Encoded(spSecret), "service principal secret"),
+		// Bootstrap token is already encoded so we don't need to
+		// encode it again here.
+		FileExcludesContentsValidator(logPath, bootstrapToken, "bootstrap token"),
+	}
 }
diff --git a/e2e/validators.go b/e2e/validators.go
@@ -125,6 +125,19 @@ func FileHasContentsValidator(fileName string, contents string) *LiveVMValidator
 	}
 }
 
+func FileExcludesContentsValidator(fileName string, contents string, contentsName string) *LiveVMValidator {
+	return &LiveVMValidator{
+		Description: fmt.Sprintf("assert %s does not contain %s", fileName, contentsName),
+		Command:     fmt.Sprintf("grep -q -F '%s' '%s'", contents, fileName),
+		Asserter: func(code, stdout, stderr string) error {
+			if code == "0" {
+				return fmt.Errorf("expected to find a file '%s' without %s but did not", fileName, contentsName)
+			}
+			return nil
+		},
+	}
+}
+
 // this function is just used to remove some bash specific tokens so we can echo the command to stdout.
 func cleanse(str string) string {
 	str = strings.Replace(str, "'", "", -1)

diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh
@@ -185,6 +185,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -195,7 +197,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+DynamicKubeletConfig/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+EnablePrivateClusterHostsConfigAgent/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"

diff --git a/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CustomData b/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+GPUDedicatedVHD/line70.sh
@@ -175,6 +175,8 @@ configureK8s() {
     chown root:root "${AZURE_JSON_PATH}"
 
     mkdir -p "/etc/kubernetes/certs"
+
+    set +x
     if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then
         echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key
     fi
@@ -185,7 +187,6 @@ configureK8s() {
         echo "${SERVICE_PRINCIPAL_FILE_CONTENT}" | base64 -d > /etc/kubernetes/sp.txt
     fi
 
-    set +x
     echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
     SP_FILE="/etc/kubernetes/sp.txt"
     SERVICE_PRINCIPAL_CLIENT_SECRET="$(cat "$SP_FILE")"