From cdf94dd4d5ca99177c09b265b5b8f7553f697bde Mon Sep 17 00:00:00 2001 From: Tomislav Plavcic Date: Wed, 20 Sep 2023 18:55:54 +0200 Subject: [PATCH] Push failed test k8s logs to S3 bucket --- Jenkinsfile | 18 ++++++++++++++++++ e2e-tests/functions | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 036ab6093a..411390be85 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -103,6 +103,21 @@ void pushLogFile(String FILE_NAME) { } } +void pushK8SLogs(String TEST_NAME) { + def LOG_FILE_PATH="e2e-tests/logs/" + def FILE_NAMES="logs_${TEST_NAME}_*" + echo "Push k8s logs to S3!" + + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: 'AMI/OVF', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) { + sh """ + S3_PATH=s3://percona-jenkins-artifactory/\$JOB_NAME/\$(git rev-parse --short HEAD)/logs/ + aws s3 ls \$S3_PATH || : + aws s3 rm \$S3_PATH --recursive --exclude "*" --include "${FILE_NAMES}" || : + aws s3 cp --quiet ${LOG_FILE_PATH} \$S3_PATH --recursive --exclude "*" --include "$FILE_NAMES" || : + """ + } +} + void popArtifactFile(String FILE_NAME) { echo "Try to get $FILE_NAME file from S3!" @@ -218,6 +233,9 @@ void runTest(Integer TEST_ID) { return true } catch (exc) { + if (currentBuild.nextBuild != null) { + pushK8SLogs("$testName") + } if (retryCount >= 1 || currentBuild.nextBuild != null) { currentBuild.result = 'FAILURE' return true diff --git a/e2e-tests/functions b/e2e-tests/functions index 4e9a7ee514..f4d97ae054 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -158,6 +158,8 @@ wait_pod() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -183,6 +185,8 @@ wait_cron() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -203,6 +207,8 @@ wait_backup_agent() { if [ $retry -ge 360 ]; then kubectl_bin logs $agent_pod -c backup-agent \ | tail -100 + collect_k8s_logs + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -230,6 +236,8 @@ wait_backup() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs + echo "Backup object psmdb-backup/${backup_name} is in ${current_state} state." echo something went wrong with operator or kubernetes cluster exit 1 @@ -289,6 +297,8 @@ wait_deployment() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -334,6 +344,7 @@ wait_restore() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs echo "Restore object restore-${backup_name} is in ${current_state} state." echo something went wrong with operator or kubernetes cluster @@ -528,6 +539,7 @@ retry() { until "$@"; do if [[ $n -ge $max ]]; then + collect_k8s_logs echo "The command '$@' has failed after $n attempts." exit 1 fi @@ -567,6 +579,7 @@ wait_for_running() { timeout=$((timeout + 1)) echo -n '.' if [[ ${timeout} -gt 1500 ]]; then + collect_k8s_logs echo echo "Waiting timeout has been reached. Exiting..." exit 1 @@ -594,6 +607,8 @@ wait_for_delete() { | grep -v 'Getting tasks for pod' \ | grep -v 'Getting pods from source' \ | tail -100 + collect_k8s_logs + echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -609,6 +624,8 @@ compare_generation() { current_generation="$(kubectl_bin get ${resource_type} "${resource_name}" -o jsonpath='{.metadata.generation}')" if [[ ${generation} != "${current_generation}" ]]; then + collect_k8s_logs + echo "Generation for ${resource_type}/${resource_name} is: ${current_generation}, but should be: ${generation}" exit 1 fi @@ -961,6 +978,7 @@ get_service_endpoint() { return fi + collect_k8s_logs exit 1 } @@ -1135,6 +1153,7 @@ wait_cluster_consistency() { until [[ "$(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.state}')" == "ready" ]]; do let retry+=1 if [ $retry -ge 32 ]; then + collect_k8s_logs echo max retry count $retry reached. something went wrong with operator or kubernetes cluster exit 1 fi @@ -1161,6 +1180,7 @@ check_backup_deletion() { retry=0 until [[ $(curl -sw '%{http_code}' -o /dev/null $path) -eq 403 ]] || [[ $(curl -sw '%{http_code}' -o /dev/null $path) -eq 404 ]]; do if [ $retry -ge 10 ]; then + collect_k8s_logs echo max retry count $retry reached. something went wrong with operator or kubernetes cluster echo "Backup was not removed from bucket -- $storage_name" exit 1 @@ -1222,6 +1242,7 @@ function get_mongod_ver_from_image() { version_info=$(run_simple_cli_inside_image ${image} 'mongod --version' | $sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g') if [[ ! ${version_info} =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]; then + collect_k8s_logs printf "No mongod version obtained from %s. Exiting" ${image} exit 1 fi @@ -1234,6 +1255,7 @@ function get_pbm_version() { local version_info=$(run_simple_cli_inside_image ${image} 'pbm-agent version' | $sed -r 's/^Version:\ (([0-9]+\.){2}[0-9]+)\ .*/\1/g') if [[ ! ${version_info} =~ ^([0-9]+\.){2}[0-9]+$ ]]; then + collect_k8s_logs printf "No pbm version obtained from %s. Exiting" ${image} exit 1 fi @@ -1274,6 +1296,28 @@ function generate_vs_json() { echo ${version_service_source} | jq '.' >${target_path} } +collect_k8s_logs() { + if [[ ${ENABLE_LOGGING} == "true" ]]; then + rm -f ${logs_dir}/logs_${test_name}_*.txt || : + + local check_namespaces="$namespace${OPERATOR_NS:+ $OPERATOR_NS}" + + for ns in $check_namespaces; do + local pods=$(kubectl_bin get pods -o name | awk -F "/" '{print $2}') + for p in $pods; do + local containers=$(kubectl_bin -n "$ns" get pod $p -o jsonpath='{.spec.containers[*].name}') + for c in $containers; do + kubectl_bin -n "$ns" logs $p -c $c > ${logs_dir}/logs_${test_name}_$p_$c.txt + echo logs saved in: ${logs_dir}/logs_${test_name}_$p_$c.txt + done + done + done + kubectl_bin get pods --all-namespaces > ${logs_dir}/logs_${test_name}_pods.txt + kubectl_bin get services --all-namespaces > ${logs_dir}/logs_${test_name}_services.txt + kubectl_bin get events --all-namespaces > ${logs_dir}/logs_${test_name}_k8s_events.txt + fi +} + check_passwords_leak() { secrets=$(kubectl_bin get secrets -o json | jq -r '.items[].data | to_entries | .[] | select(.key | (contains("_PASSWORD"))) | .value') echo secrets=$secrets