CLOUD-789 - add test to check password leaks in the logs

e2e-tests/conf/secrets.yml

@@ -6,10 +6,10 @@ type: Opaque
 data:
   root: cm9vdF9wYXNzd29yZA==
   xtrabackup: YmFja3VwX3Bhc3N3b3Jk
-  monitor: bW9uaXRvcg==
+  monitor: bW9uaXRvcl9wYXNzd29yZA==
   clustercheck: Y2x1c3RlcmNoZWNrcGFzc3dvcmQ=
   proxyadmin: YWRtaW5fcGFzc3dvcmQ=
-  pmmserver: YWRtaW4=
+  pmmserver: cG1tc2VydmVyX3Bhc3N3b3Jk
   operator: b3BlcmF0b3JhZG1pbg==
   replication: cmVwbF9wYXNzd29yZA==
 ---

e2e-tests/conf/secrets_without_tls.yml

@@ -6,9 +6,9 @@ type: Opaque
 data:
   root: cm9vdF9wYXNzd29yZA==
   xtrabackup: YmFja3VwX3Bhc3N3b3Jk
-  monitor: bW9uaXRvcg==
+  monitor: bW9uaXRvcl9wYXNzd29yZA==
   clustercheck: Y2x1c3RlcmNoZWNrcGFzc3dvcmQ=
   proxyadmin: YWRtaW5fcGFzc3dvcmQ=
-  pmmserver: YWRtaW4=
+  pmmserver: cG1tc2VydmVyX3Bhc3N3b3Jk
   operator: b3BlcmF0b3JhZG1pbg==
   replication: cmVwbF9wYXNzd29yZA==

e2e-tests/functions

@@ -1518,3 +1518,57 @@ function check_backup_deletion() {
 		((retry += 1))
 	done
 }
+
+check_passwords_leak() {
+	local secrets
+	local passwords
+	local pods
+
+	secrets=$(kubectl_bin get secrets -o json | jq -r '.items[].data | to_entries | .[] | select(.key | (endswith(".crt") or endswith(".key") or endswith(".pub") or endswith(".pem") or endswith(".p12") or contains("release") or contains("namespace") or contains("AWS_ACCESS_KEY_ID") or contains("AZURE_STORAGE_ACCOUNT_NAME")) | not) | .value')
+	echo secrets=$secrets
+
+	passwords="$(for i in $secrets; do base64 -d <<< $i; echo; done) $secrets"
+	echo passwords=$passwords
+
+	pods=$(kubectl_bin get pods -o name | awk -F "/" '{print $2}')
+	echo pods=$pods
+
+	collect_logs() {
+		local containers
+		local count
+
+		NS=$1
+		for p in $pods; do
+			if [[ $p == "monitoring-0" ]]; then
+				continue
+			fi
+			containers=$(kubectl_bin -n "$NS" get pod $p -o jsonpath='{.spec.containers[*].name}')
+			for c in $containers; do
+				# temporary, because of: https://jira.percona.com/browse/PMM-8357
+				if [[ $c =~ "pmm" ]]; then
+					continue
+				fi
+				kubectl_bin -n "$NS" logs $p -c $c > ${tmp_dir}/logs_output-$p-$c.txt
+				echo logs saved in: ${tmp_dir}/logs_output-$p-$c.txt
+				for pass in $passwords; do
+					echo trying password: $pass
+					count=$(grep -c --fixed-strings -- "$pass" ${tmp_dir}/logs_output-$p-$c.txt || :)
+					if [[ $count != 0 ]]; then
+						echo leaked password $pass is found in log ${tmp_dir}/logs_output-$p-$c.txt
+						echo the content of file ${tmp_dir}/logs_output-$p-$c.txt is:
+						echo =========================================================
+						cat ${tmp_dir}/logs_output-$p-$c.txt
+						false
+					fi
+				done
+			done
+			echo
+		done
+	}
+
+	collect_logs $namespace
+	if [ -n "$OPERATOR_NS" ]; then
+		pods=$(kubectl_bin -n "${OPERATOR_NS}" get pods -o name | awk -F "/" '{print $2}')
+		collect_logs $OPERATOR_NS
+	fi
+}

e2e-tests/haproxy/run

@@ -66,6 +66,9 @@ main() {
 	wait_for_running "$cluster-pxc" 3
 	check_haproxy_writer
 
+	desc 'check for passwords leak'
+	check_passwords_leak
+
 	desc 'delete active writer and checking all haproxy pods still point to the same writer'
 	desc 'fail pxc-pod-0 pod for 60s'
 	fail_pod $cluster-pxc-0

e2e-tests/init-deploy/run

@@ -33,19 +33,19 @@ compare_kubectl pdb/$cluster-proxysql
 
 desc 'check if MySQL users created'
 compare_mysql_user "-h $cluster-pxc -uroot -proot_password"
-compare_mysql_user "-h $cluster-pxc -umonitor -pmonitor"
+compare_mysql_user "-h $cluster-pxc -umonitor -pmonitor_password"
 compare_mysql_user "-h $cluster-pxc -uproxyuser -ps3cret"
 compare_mysql_user_local "-uxtrabackup -pbackup_password" "$cluster-pxc-0" "" "pxc"
 compare_mysql_user_local "-uclustercheck -pclustercheckpassword" "$cluster-pxc-0" "" "pxc"
 desc "check that pmm server user don't have access"
-compare_mysql_user "-h $cluster-pxc -upmmserver -pmonitor"
+compare_mysql_user "-h $cluster-pxc -upmmserver -ppmmserver_password"
 
 desc 'check if ProxySQL users created'
 compare_mysql_user "-h $cluster-proxysql -uroot -proot_password"
-compare_mysql_user "-h $cluster-proxysql -umonitor -pmonitor"
+compare_mysql_user "-h $cluster-proxysql -umonitor -pmonitor_password"
 desc "check that pmm server user don't have access"
 compare_mysql_user "-h $cluster-proxysql -uproxyuser -ps3cret" "-proxysql"
-compare_mysql_user "-h $cluster-proxysql -upmmserver -pmonitor" "-proxysql"
+compare_mysql_user "-h $cluster-proxysql -upmmserver -ppmmserver_password" "-proxysql"
 
 desc 'write data directly, read from all'
 run_mysql \
@@ -98,7 +98,10 @@ compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster3-pxc-0.$c
 compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster3-pxc-1.$cluster3-pxc -uroot -proot_password"
 compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster3-pxc-2.$cluster3-pxc -uroot -proot_password"
 compare_mysql_cmd "max_allowed_packet-2" "SELECT @@max_allowed_packet;" "-h $cluster3-pxc-0.$cluster3-pxc -uroot -proot_password"
-kubectl_bin delete -f $test_dir/conf/$cluster3.yml
 
+desc 'check for passwords leak'
+check_passwords_leak
+
+kubectl_bin delete -f $test_dir/conf/$cluster3.yml
 destroy $namespace
 desc "test passed"

e2e-tests/monitoring-2-0/compare/agents-list.json

@@ -121,7 +121,6 @@
     "service_type": "postgresql",
     "database_name": "postgres",
     "node_name": "pmm-server",
-    "cluster": "pmm-server-postgresql",
     "address": "127.0.0.1",
     "port": 5432,
     "agents": [

e2e-tests/monitoring-2-0/run

@@ -11,11 +11,8 @@ get_mgmnt_service_list() {
 	local api_key=$1
 	local endpoint=$2
 	local namespace=$3
-	cat >${tmp_dir}/payload.json <<EOF
-{}
-EOF
 
-	curl -s -k -H "Authorization: Bearer ${api_key}" -X POST -d @payload.json "https://$endpoint/v1/management/Service/List" \
+	curl -s -k -H "Authorization: Bearer ${api_key}" -X POST "https://$endpoint/v1/management/Service/List" \
 		| jq 'walk(if type=="object" then with_entries(select(.key | test("service_id|node_id|agent_id|created_at|updated_at|service_name") | not)) else . end)' \
 		| jq 'walk(if type == "array" then sort_by(.agent_type) else . end)' \
 			>${tmp_dir}/active_pmm_agents.json
@@ -71,8 +68,7 @@ else
 	helm install monitoring --set imageTag=$IMAGE_PMM_SERVER_TAG --set imageRepo=$IMAGE_PMM_SERVER_REPO --set platform=$platform https://percona-charts.storage.googleapis.com/pmm-server-${PMM_SERVER_VER}.tgz
 fi
 kubectl_bin wait --for=condition=Ready pod/${cluster}-0 --timeout=120s
-SERVICE="postgres"
-until kubectl_bin exec monitoring-0 -- bash -c "pgrep -x $SERVICE >/dev/null"; do
+until kubectl_bin exec monitoring-0 -- bash -c "ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null"; do
 	echo "Retry $retry"
 	sleep 5
 	let retry+=1
@@ -160,6 +156,10 @@ if [[ -n ${OPENSHIFT} ]]; then
 		oc delete rolebinding pmm-pxc-operator-namespace-only
 	fi
 fi
+
+desc 'check for passwords leak'
+check_passwords_leak
+
 helm uninstall monitoring
 destroy $namespace
 desc "test passed"

e2e-tests/pitr/run

@@ -95,6 +95,9 @@ main() {
 
 	run_backup "$cluster" "on-pitr-minio"
 
+	desc 'check for passwords leak'
+	check_passwords_leak
+
 	write_test_data "$cluster"
 
 	desc 'show binlog events'
@@ -134,6 +137,10 @@ main() {
 
 	run_recovery_check_pitr "$cluster" "restore-on-pitr-minio-gtid" "on-pitr-minio" "select-2" "" "" "$gtid"
 	desc "done gtid type"
+
+	desc 'check for passwords leak'
+	check_passwords_leak
+
 	sleep 60
 	if [[ $(kubectl get pxc-backup on-pitr-minio -o jsonpath='{.status.conditions}' | grep -c 'Binlog with GTID set') -eq 1 ]]; then
 		echo "Binlog gap detected"