update victoriametrics availability check during exporter startup

process; adds a polling loop to retry if server is not available on first try based on experiences using Lustre as a datastore at ORNL Signed-off-by: Karl W. Schulz <[email protected]>
AMDResearch · Jan 17, 2025 · dd6fb95 · dd6fb95
1 parent cd854eb
commit dd6fb95
Showing 1 changed file with 19 additions and 7 deletions.
diff --git a/omnistat/standalone.py b/omnistat/standalone.py
@@ -96,14 +96,26 @@ def __init__(self, args, config):
             logging.error("[ERROR]: Please set data_frequency_mins >= 1 minute (%s)" % self.__pushFrequencyMins)
             sys.exit(1)
 
-        # verify victoriaURL is accessible
-        failed = False
-        try:
-            response = requests.get(self.__victoriaURL)
-            if response.status_code != 204:
+        # verify victoriaURL is operational and ready to receive queries (poll for a bit if necessary)
+        failed = True
+        delay_start = 0.05
+        testURL = f"http://{args.endpoint}:{args.port}/ready"
+        for iter in range(1, 25):
+            try:
+                response = requests.get(testURL)
+                logging.debug("VM ready response = %s" % response)
+                if response.status_code != 200:
+                    failed = True
+                else:
+                    failed = False
+                    break
+            except requests.exceptions.RequestException as e:
+                logging.debug(e)
                 failed = True
-        except requests.exceptions.RequestException as e:
-            failed = True
+
+            delay = delay_start * iter
+            logging.info("Retrying VM endpoint (sleeping for %.2f sec)" % (delay))
+            time.sleep(delay)
 
         if failed:
             logging.error("")