From f2ecee80ce247855104fa0c133c96d1fb94e5004 Mon Sep 17 00:00:00 2001
From: Jacob Salmela <jacob.salmela@hpe.com>
Date: Fri, 7 Feb 2025 10:27:25 -0600
Subject: [PATCH] CASMINST-7173 add retry logic to postgres_clusters_running.sh

detected in auto-triage, this script failed, but worked upon manual
execution by a human.  this adds a simple retry logic as an additional
safety net to help prevent false auto-triage tickets.

Signed-off-by: Jacob Salmela <jacob.salmela@hpe.com>
---
 .../scripts/postgres_clusters_running.sh      | 30 ++++++++++++++-----
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/goss-testing/scripts/postgres_clusters_running.sh b/goss-testing/scripts/postgres_clusters_running.sh
index 889bbcc1..304fabeb 100644
--- a/goss-testing/scripts/postgres_clusters_running.sh
+++ b/goss-testing/scripts/postgres_clusters_running.sh
@@ -22,13 +22,29 @@
 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 #
-failFlag=0
-postgresStatuses="$(kubectl get postgresql -A -o jsonpath='{.items[*].status.PostgresClusterStatus}')"
-for status in $postgresStatuses
+maxRetries=3
+retryCount=0
+retryDelay=2
+
+while [[ $retryCount -lt $maxRetries ]]
 do
-    if [[ $status != "Running" && $status != "Updating" ]]; then failFlag=1; fi
+    failFlag=0
+    postgresStatuses="$(kubectl get postgresql -A -o jsonpath='{.items[*].status.PostgresClusterStatus}')"
+    for status in $postgresStatuses
+    do
+        if [[ $status != "Running" && $status != "Updating" ]]; then failFlag=1; fi
+    done
+
+    if [[ $failFlag -eq 0 ]]; then
+        result="PASS"
+        break
+    else
+        result="FAIL"
+    fi
+
+    retryCount=$((retryCount + 1))
+    sleep "$retryDelay"
 done
 
-if [[ $failFlag -eq 0 ]]; then echo "PASS"; exit 0;
-else exit 1
-fi
+echo "$result"
+if [[ $result == "PASS" ]]; then exit 0; else exit 1; fi