PMM-11178 retry errors during PBM restore check (#1478)

* PMM-11178 retry describe restores error

* reset on success

* move else block outside

agent/runner/jobs/pbm_helpers.go

@@ -328,6 +328,7 @@ func waitForPBMRestore(ctx context.Context, l logrus.FieldLogger, dbURL *url.URL
 	ticker := time.NewTicker(statusCheckInterval)
 	defer ticker.Stop()
 
+	maxRetryCount := 5
 	for {
 		select {
 		case <-ticker.C:
@@ -338,8 +339,16 @@ func waitForPBMRestore(ctx context.Context, l logrus.FieldLogger, dbURL *url.URL
 				err = execPBMCommand(ctx, dbURL, &info, "describe-restore", name)
 			}
 			if err != nil {
-				return errors.Wrap(err, "failed to get restore status")
+				if maxRetryCount > 0 {
+					maxRetryCount--
+					l.Warnf("PMM failed to get backup restore status and will retry: %s", err)
+					continue
+				} else {
+					return errors.Wrap(err, "failed to get restore status")
+				}
 			}
+			// reset maxRetryCount if we were able to successfully get the current restore status
+			maxRetryCount = 5
 
 			switch info.Status {
 			case "done":