Download all sources for dist-git-source

Document common options on a single place
Option to just download sources (no extractions)
Fix 'url' behaviour for 'shell'
Move 'distgit_download' to utils

spec/plans/discover.fmf

@@ -68,22 +68,9 @@ description: |
         keep-git-metadata
             By default the ``.git`` directory is removed to save
             disk space. Set to ``true`` to sync the git metadata
-            to guest as well.
+            to guest as well. Implicit if ``dist-git-source`` is used.
 
-        There is a support for discovering tests from extracted
-        (rpm) sources. Needs to run on top of the supported
-        DistGit (Fedora, CentOS), ``url`` can be used to point to
-        such repository, ``path`` denotes path to the metadata
-        tree root within extracted sources. At this moment no
-        patches are applied, only tarball extraction happens.
 
-        dist-git-source
-            Set to ``true`` to enable extracting sources.
-        dist-git-type
-            The DistGit auto detection is based on git remotes.
-            Use this option to specify it directly. At least
-            ``Fedora`` and ``CentOS`` are supported as values,
-            help will print all possible values.
 
     example:
       - |
@@ -185,20 +172,27 @@ description: |
             specify ``reference/<branch>`` to compare to a branch
             from the repository specified in ``modified-url``.
 
-        There is a support for discovering tests from extracted
-        (rpm) sources. Needs to run on top of the supported
-        DistGit (Fedora, CentOS), ``url`` can be used to point to
-        such repository, ``path`` denotes path to the metadata
-        tree root within extracted sources. At this moment no
-        patches are applied, only tarball extraction happens.
+        Additional dist-git options are available
+
+        dist-git-init
+            Set to True to initialize fmf root inside extracted sources at
+            ``dist-git-extract`` location or top directory. To be used
+            when the sources does not contain fmf root but do included fmf
+            files.
+
+        dist-git-remove-fmf-root
+            Set to True to remove any fmf root present in the sources.
+            ``dist-git-init`` can be used to create it later at desired location.
+
+        dist-git-merge
+            Set to True to combine fmf root from the sources and fmf root from plan.
+            For example when ``dist-git-download-only`` is used this option allows to
+            find tests defined within plans repo.
+
+        dist-git-extract
+            Path specifying what should be copied from the sources.
+            Defaults to top fmf root or top directory ("/").
 
-        dist-git-source
-            Set to ``true`` to enable extracting sources.
-        dist-git-type
-            The DistGit auto detection is based on git remotes.
-            Use this option to specify it directly. At least
-            ``Fedora`` and ``CentOS`` are supported as values,
-            help will print all possible values.
 
         .. _fmf identifier: https://fmf.readthedocs.io/en/latest/concept.html#identifiers
         .. _Flexible Metadata Format: https://fmf.readthedocs.io/
@@ -236,6 +230,60 @@ description: |
     link:
       - implemented-by: /tmt/steps/discover/fmf.py
 
+/dist-git-source:
+    summary: Download rpm sources
+    description: |
+      There is a support to download sources of the rpm.
+      Plan using the option has to be defined in DistGit(Fedora, CentOS)
+      or ``url`` option needs to point top such repository.
+      At this moment no patches are applied, only sources are downloaded
+      and tarballs extracted.
+
+      All source files are available for further use in the directory in
+      ``TMT_SOURCE_DIR`` test variable.
+
+      Plugins might define more options, see their documentation. Globally available
+      options are
+
+        dist-git-type
+          Use the provided DistGit handler instead of the auto detection. Useful
+          when running from forked repositories.
+
+        dist-git-fetch-only
+          When ``true`` no tarballs are extracted. Saves space if one wants to apply
+          patches later on their own.
+    example:
+      - |
+        # Download & extract sources from another repo, print single file as a test
+        discover:
+          how: shell
+          url: https://src.fedoraproject.org/rpms/tmt
+          dist-git-source: true
+          tests:
+          - name: /print/pyproject
+            test: cat $TMT_SOURCE_DIR/tmt-*/pyproject.toml
+      - |
+        # Just download sources, test is reponsible for rpmbuild and running tests
+        discover:
+          how: shell
+          dist-git-source: true
+          dist-git-fetch-only: true
+          tests:
+          - name: /unit
+            test: >
+              rpmbuild -bp --define "_sourcedir $TMT_SOURCE_DIR"
+                --define "_builddir $TMT_SOURCE_DIR/BUILD"
+                $TMT_SOURCE_DIR/*.spec &&
+              cd $TMT_SOURCE_DIR/BUILD/* &&
+              make test
+            require:
+            - rpm-build
+
+    link:
+      - implemented-by: /tmt/steps/discover/fmf.py
+      - implemented-by: /tmt/steps/discover/shell.py
+
+
 /where:
     summary: Execute tests on selected guests
     description: |

tests/discover/distgit.sh

@@ -44,6 +44,10 @@ rlJournalStart
             touch $tmp/outsider
             rlRun "tar czvf $tmp/simple-1.tgz --directory $tmp simple-1 outsider"
             rlRun "rm -rf $tmp/simple-1 outsider"
+            # Other files tests can ask for
+            for f in file.tgz.asc file.key something.gem; do
+                rlRun "touch $SERVER_DIR/$f"
+            done
         )
 
         # prepare unit-no-tmt (e.g. pytest files inside tarball)
@@ -59,7 +63,7 @@ rlJournalStart
         rlRun "popd"
     rlPhaseEnd
 
-
+if false; then
     ### discover -h fmf ###
 
 for value in explicit auto; do
@@ -162,25 +166,19 @@ done
 
 
 
-    rlPhaseStartTest "non-tar files in sources are ignored"
+    rlPhaseStartTest "all source files are downloaded"
         rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory"
         rlRun 'pushd $tmp'
 
         rlRun "git init" # should be git
         rlRun "tmt init" # should has fmf tree
 
-        # https://github.com/teemtee/tmt/issues/1055
-        # https://github.com/teemtee/tmt/issues/1949
-
         (
             echo simple-1.tgz
-            echo IGNORED file.tgz.asc
-            echo IGNORED file.key
-            echo IGNORED file.sign
-            echo IGNORED cockpit.css.gz
+            echo file.tgz.asc
+            echo file.key
+            echo something.gem
         ) > $MOCK_SOURCES_FILENAME
-        # only simple-1.tgz should be downloaded so all other would fail with
-        # File not found for url: http://localhost:9000/IGNORED
 
         rlRun -s 'tmt run --id /var/tmp/tmt/XXX --scratch plans --default \
              discover -vvv -ddd --how fmf --dist-git-source \
@@ -404,7 +402,7 @@ done
 
     ### discover -h shell ###
 
-    rlPhaseStartTest "shell with merge (tmt for plan only)"
+    rlPhaseStartTest "shell always merges the plan's git"
         rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory"
         rlRun 'pushd $tmp'
 
@@ -447,7 +445,47 @@ EOF
         rlRun "rm $rlRun_LOG"
         rlRun "rm -rf $tmp"
     rlPhaseEnd
+fi
+
+rlPhaseStartTest "shell with fetch-only"
+        rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory"
+        rlRun 'pushd $tmp'
 
+        rlRun "git init"
+        echo unit-no-tmt.tgz > $MOCK_SOURCES_FILENAME
+
+        rlRun "tmt init"
+        cat <<EOF > plans.fmf
+discover:
+    how: shell
+    tests:
+    -   name: /tarball is there
+        test: ls \$TMT_SOURCE_DIR
+    dist-git-source: true
+    dist-git-type: TESTING
+    dist-git-fetch-only: true
+provision:
+    how: local
+execute:
+    how: tmt
+EOF
+
+        WORKDIR=/var/tmp/tmt/XXX
+        WORKDIR_SOURCE=$WORKDIR/plans/discover/default-0/source
+
+        rlRun -s "tmt run --keep --id $WORKDIR --scratch -vvv"
+
+        # Tarball was not extracted
+        rlAssertNotExists $WORKDIR_SOURCE/foo-123/all_in_one
+        # But downloaded
+        rlAssertExists $WORKDIR_SOURCE/unit-no-tmt.tgz
+        rlAssertExists $WORKDIR_SOURCE/$MOCK_SOURCES_FILENAME
+
+
+        rlRun "popd"
+        rlRun "rm $rlRun_LOG"
+        rlRun "rm -rf $tmp"
+    rlPhaseEnd
 
     rlPhaseStartCleanup
         echo $SERVER_PID

tmt/schemas/common.yaml

@@ -109,20 +109,22 @@ definitions:
     # name: true
     # type: true
 
-  # https://tmt.readthedocs.io/en/stable/spec/plans.html#fmf
-  # https://tmt.readthedocs.io/en/stable/spec/plans.html#shell
+  # https://tmt.readthedocs.io/en/stable/spec/plans.html#dist-git-source
   dist-git-source:
     type: boolean
 
-  # https://tmt.readthedocs.io/en/stable/spec/plans.html#fmf
-  # https://tmt.readthedocs.io/en/stable/spec/plans.html#shell
+  # https://tmt.readthedocs.io/en/stable/spec/plans.html#dist-git-source
   dist-git-type:
     type: string
     enum:
       - centos
       - fedora
       - rhel
 
+  # https://tmt.readthedocs.io/en/stable/spec/plans.html#dist-git-source
+  dist-git-fetch-only:
+    type: boolean
+
   # https://tmt.readthedocs.io/en/stable/spec/tests.html#duration
   # https://tmt.readthedocs.io/en/stable/spec/plans.html#shell
   duration:

tmt/steps/discover/__init__.py

@@ -17,7 +17,7 @@
 from tmt.options import option
 from tmt.plugins import PluginRegistry
 from tmt.steps import Action
-from tmt.utils import Command, GeneralError, Path, field, flatten, key_to_option
+from tmt.utils import GeneralError, Path, field, flatten, key_to_option
 
 
 @dataclasses.dataclass
@@ -26,7 +26,7 @@ class DiscoverStepData(tmt.steps.WhereableStepData, tmt.steps.StepData):
         default=False,
         option='--dist-git-source',
         is_flag=True,
-        help='Extract DistGit sources.'
+        help='Download and possibly extract DistGit sources.'
         )
 
     # TODO: use enum!
@@ -37,6 +37,13 @@ class DiscoverStepData(tmt.steps.WhereableStepData, tmt.steps.StepData):
         help='Use the provided DistGit handler instead of the auto detection.'
         )
 
+    dist_git_fetch_only: bool = field(
+        default=False,
+        option="--dist-git-fetch-only",
+        is_flag=True,
+        help="Just download all sources, do not extract them.",
+        )
+
 
 DiscoverStepDataT = TypeVar('DiscoverStepDataT', bound=DiscoverStepData)
 
@@ -89,37 +96,22 @@ def tests(
         raise NotImplementedError
 
     def extract_distgit_source(
-            self, distgit_dir: Path, target_dir: Path, handler_name: Optional[str] = None) -> None:
+            self, distgit_dir: Path, target_dir: Path, handler_name: Optional[str] = None,
+            fetch_only: bool = False) -> None:
         """
-        Extract source tarball into target_dir
+        Download sources to the target_dir and possibly extract the tarballs
 
-        distgit_dir is path to the DistGit repository.
-        Source tarball is discovered from the 'sources' file content.
+        distgit_dir is path to the DistGit repository
+        Set fetch_only if extraction should not happen
         """
-        if handler_name is None:
-            output = self.run(
-                Command("git", "config", "--get-regexp", '^remote\\..*.url'),
-                cwd=distgit_dir)
-            if output.stdout is None:
-                raise tmt.utils.GeneralError("Missing remote origin url.")
-
-            remotes = output.stdout.split('\n')
-            handler = tmt.utils.get_distgit_handler(remotes=remotes)
-        else:
-            handler = tmt.utils.get_distgit_handler(usage_name=handler_name)
-        for url, source_name in handler.url_and_name(distgit_dir):
-            if not handler.re_supported_extensions.search(source_name):
-                continue
-            self.debug(f"Download sources from '{url}'.")
-            with tmt.utils.retry_session() as session:
-                response = session.get(url)
-            response.raise_for_status()
-            target_dir.mkdir(exist_ok=True, parents=True)
-            with open(target_dir / source_name, 'wb') as tarball:
-                tarball.write(response.content)
-            self.run(
-                Command("tar", "--auto-compress", "--extract", "-f", source_name),
-                cwd=target_dir)
+        tmt.utils.distgit_download(
+            distgit_dir=distgit_dir,
+            target_dir=target_dir,
+            handler_name=handler_name,
+            fetch_only=fetch_only,
+            caller=self,
+            logger=self._logger
+            )
 
     def log_import_plan_details(self) -> None:
         """