Retry the git clone action multiple times

Signed-off-by: Vector Li <[email protected]>
Co-authored-by: Petr Šplíchal <[email protected]>

Author: Vector Li

docs/overview.rst

@@ -419,6 +419,17 @@ TMT_GIT_CREDENTIALS_URL_<suffix>, TMT_GIT_CREDENTIALS_VALUE_<suffix>
 __ https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#clone-repository-using-personal-access-token
 __ https://github.blog/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/
 
+TMT_GIT_CLONE_ATTEMPTS
+    The maximum number of retries to clone a git repository if it
+    fails. By default, 3 attempts are done.
+
+TMT_GIT_CLONE_INTERVAL
+    The interval (in seconds) to retry clonning a git repository
+    again, 10 seconds by default.
+
+TMT_GIT_CLONE_TIMEOUT
+    Overall maximum time in seconds to clone a git repository. By
+    default, the limit is not set.
 
 .. _step-variables:
 

tmt/base.py

@@ -2327,7 +2327,10 @@ def import_plan(self) -> Optional['Plan']:
             if destination.exists():
                 self.debug(f"Seems that '{destination}' has been already cloned.", level=3)
             else:
-                tmt.utils.git_clone(plan_id.url, destination, self)
+                tmt.utils.git_clone(
+                    url=plan_id.url,
+                    destination=destination,
+                    logger=self._logger)
             if plan_id.ref:
                 # Attempt to evaluate dynamic reference
                 try:
@@ -2355,7 +2358,12 @@ def import_plan(self) -> Optional['Plan']:
                 self.debug(f"Not enough data to evaluate dynamic ref '{plan_id.ref}', "
                            "going to clone the repository to read dynamic ref definition.")
                 with tempfile.TemporaryDirectory() as tmpdirname:
-                    git_clone(str(plan_id.url), Path(tmpdirname), self, None, shallow=True)
+                    git_clone(
+                        url=str(plan_id.url),
+                        destination=Path(tmpdirname),
+                        shallow=True,
+                        env=None,
+                        logger=self._logger)
                     self.run(Command(
                         'git', 'checkout', 'HEAD', str(plan_id.ref)[1:]),
                         cwd=Path(tmpdirname))

tmt/libraries/beakerlib.py

@@ -217,9 +217,13 @@ def fetch(self) -> None:
                     with TemporaryDirectory() as tmp:
                         assert self.url is not None  # narrow type
                         try:
-                            tmt.utils.git_clone(self.url, Path(tmp), self.parent,
-                                                env={"GIT_ASKPASS": "echo"}, shallow=True)
-                        except tmt.utils.RunError:
+                            tmt.utils.git_clone(
+                                url=self.url,
+                                destination=Path(tmp),
+                                shallow=True,
+                                env={"GIT_ASKPASS": "echo"},
+                                logger=self._logger)
+                        except (tmt.utils.RunError, tmt.utils.RetryError):
                             self.parent.debug(f"Repository '{self.url}' not found.")
                             self._nonexistent_url.add(self.url)
                             raise LibraryError
@@ -257,8 +261,11 @@ def fetch(self) -> None:
                     # minimize data transfers if ref is not provided
                     if not clone_dir.exists():
                         tmt.utils.git_clone(
-                            self.url, clone_dir, self.parent,
-                            env={"GIT_ASKPASS": "echo"}, shallow=self.ref is None)
+                            url=self.url,
+                            destination=clone_dir,
+                            shallow=self.ref is None,
+                            env={"GIT_ASKPASS": "echo"},
+                            logger=self._logger)
 
                     # Detect the default branch from the origin
                     try:
@@ -339,7 +346,7 @@ def fetch(self) -> None:
                     self._merge_metadata(library_path, local_library_path)
                     # Copy fmf metadata
                     shutil.copytree(self.path / '.fmf', directory / '.fmf', dirs_exist_ok=True)
-            except (tmt.utils.RunError, tmt.utils.GitUrlError) as error:
+            except (tmt.utils.RunError, tmt.utils.RetryError, tmt.utils.GitUrlError) as error:
                 assert self.url is not None
                 # Fallback to install during the prepare step if in rpm format
                 if self.format == 'rpm':

tmt/steps/discover/fmf.py

@@ -357,7 +357,11 @@ def assert_git_url(plan_name: Optional[str] = None) -> None:
             # Shallow clone to speed up testing and
             # minimize data transfers if ref is not provided
             tmt.utils.git_clone(
-                url, self.testdir, self, env={"GIT_ASKPASS": "echo"}, shallow=ref is None)
+                url=url,
+                destination=self.testdir,
+                shallow=ref is None,
+                env={"GIT_ASKPASS": "echo"},
+                logger=self._logger)
             git_root = self.testdir
         # Copy git repository root to workdir
         else:

tmt/steps/discover/shell.py

@@ -275,9 +275,9 @@ def fetch_remote_repository(
         tmt.utils.git_clone(
             url=url,
             destination=testdir,
-            common=self,
+            shallow=ref is None,
             env={"GIT_ASKPASS": "echo"},
-            shallow=ref is None)
+            logger=self._logger)
 
         # Resolve possible dynamic references
         try:

tmt/utils.py

@@ -195,6 +195,10 @@ def is_pidfile(cls, exit_code: Optional[int]) -> bool:
 DEFAULT_WAIT_TICK: float = 30.0
 DEFAULT_WAIT_TICK_INCREASE: float = 1.0
 
+# Defaults for GIT attempts and interval
+DEFAULT_GIT_CLONE_ATTEMPTS: int = 3
+DEFAULT_GIT_CLONE_INTERVAL: int = 10
+
 # A stand-in variable for generic use.
 T = TypeVar('T')
 
@@ -1777,7 +1781,7 @@ class RetryError(GeneralError):
     """ Retries unsuccessful """
 
     def __init__(self, label: str, causes: list[Exception]) -> None:
-        super().__init__(f"Retries of {label} unsuccessful.", causes)
+        super().__init__(f"Retries of '{label}' unsuccessful.", causes)
 
 
 # Step exceptions
@@ -4868,41 +4872,88 @@ def distgit_download(
 
 
 def git_clone(
+        *,
         url: str,
         destination: Path,
-        common: Common,
-        env: Optional[EnvironmentType] = None,
         shallow: bool = False,
         can_change: bool = True,
-        ) -> CommandOutput:
-    """
-    Git clone url to destination, retry without shallow if necessary
+        env: Optional[EnvironmentType] = None,
+        attempts: Optional[int] = None,
+        interval: Optional[int] = None,
+        timeout: Optional[int] = None,
+        logger: tmt.log.Logger) -> CommandOutput:
+    """
+    Clone git repository from provided url to the destination directory
+
+    :param url: Source URL of the git repository.
+    :param destination: Full path to the destination directory.
+    :param shallow: For ``shallow=True`` first try to clone repository
+        using ``--depth=1`` option. If not successful clone repo with
+        the whole history.
+    :param can_change: URL can be modified with hardcoded rules. Use
+        ``can_change=False`` to disable rewrite rules.
+    :param env: Environment provided to the ``git clone`` process.
+    :param attempts: Number of tries to call the function.
+    :param interval: Amount of seconds to wait before a new try.
+    :param timeout: Overall maximum time in seconds to clone the repo.
+    :param logger: A Logger instance to be used for logging.
+    :returns: Command output, bundled in a :py:class:`CommandOutput` tuple.
+    """
+
+    def get_env(env: str, default_value: Optional[int]) -> Optional[int]:
+        """ Check environment variable, convert to integer """
+        value = os.getenv(env, None)
+        if value is None:
+            return default_value
+        try:
+            return int(value)
+        except ValueError:
+            raise GeneralError(f"Invalid '{env}' value, should be 'int', got '{value}'.")
 
-    For shallow=True attempt to clone repository using --depth=1 option first.
-    If not successful attempt to clone whole repo.
+    def clone_the_repo(
+            url: str,
+            destination: Path,
+            shallow: bool = False,
+            env: Optional[EnvironmentType] = None,
+            timeout: Optional[int] = None) -> CommandOutput:
+        """ Clone the repo, handle history depth """
 
-    Common instance is used to run the command for appropriate logging.
-    Environment is updated by 'env' dictionary.
+        depth = ['--depth=1'] if shallow else []
+        return Command('git', 'clone', *depth, url, destination).run(
+            cwd=Path('/'), env=env, timeout=timeout, logger=logger)
 
-    Url can be modified with hardcode rules unless can_change=False is set.
-    """
-    depth = ['--depth=1'] if shallow else []
+    timeout = timeout or get_env('TMT_GIT_CLONE_TIMEOUT', None)
+    attempts = attempts or cast(int, get_env('TMT_GIT_CLONE_ATTEMPTS', DEFAULT_GIT_CLONE_ATTEMPTS))
+    interval = interval or cast(int, get_env('TMT_GIT_CLONE_INTERVAL', DEFAULT_GIT_CLONE_INTERVAL))
 
+    # Update url only once
     if can_change:
         url = clonable_git_url(url)
-    try:
-        return common.run(
-            Command(
-                'git', 'clone',
-                *depth,
-                url, destination
-                ), env=env)
-    except RunError:
-        if not shallow:
-            # Do not retry if shallow was not used
-            raise
-        # Git server might not support shallow cloning, try again (do not modify url)
-        return git_clone(url, destination, common, env, shallow=False, can_change=False)
+
+    # Do an extra shallow clone first
+    if shallow:
+        try:
+            return clone_the_repo(
+                shallow=True,
+                url=url,
+                destination=destination,
+                env=env,
+                timeout=timeout)
+        except RunError:
+            logger.debug(f"Shallow clone of '{url}' failed, let's try with the full history.")
+
+    # Finish with whatever number attempts requested (deep)
+    return retry(
+        func=clone_the_repo,
+        attempts=attempts,
+        interval=interval,
+        label=f"git clone {url} {destination}",
+        url=url,
+        destination=destination,
+        shallow=False,
+        env=env,
+        timeout=timeout,
+        logger=logger)
 
 
 # ignore[type-arg]: base class is a generic class, but we cannot list its parameter type, because