Preserve order of the shell discover tests

Tests defined using the `shell` discover method should be executed
in the very same order as listed in the discover step config.

Author: psss

docs/releases.rst

@@ -5,6 +5,15 @@
 ======================
 
 
+tmt-1.41.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Tests defined using the :ref:`/plugins/discover/shell` discover
+method are now executed in the exact order as listed in the config
+file. This fixes a problem which has been introduced in the recent
+``fmf`` update.
+
+
 tmt-1.40.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

tests/discover/order.sh

@@ -177,6 +177,24 @@ EOF
 EOF
     run_test
 
+    # Tests defined using the 'shell' discover method should be
+    # executed in the order provided in the config file
+
+    plan="/shell-preserve-order"
+
+    cat > $tmp/EXPECTED-DISCOVERY <<EOF
+/tests/one
+/tests/two
+/tests/three
+EOF
+
+    cat > $tmp/EXPECTED-EXECUTION <<EOF
+/guest/default-0/tests/one-1
+/guest/default-0/tests/two-2
+/guest/default-0/tests/three-3
+EOF
+    run_test
+
     rlPhaseStartCleanup
         rlRun 'rm -rf $tmp' 0 "Remove tmp directory"
     rlPhaseEnd

tests/discover/order/plans.fmf

@@ -90,3 +90,15 @@ finish:
         test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
       - name: /order-default
         test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+
+/shell-preserve-order:
+    summary: Tests should run in the provided order
+    discover:
+        how: shell
+        tests:
+          - name: /tests/one
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+          - name: /tests/two
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order
+          - name: /tests/three
+            test: echo $TMT_TEST_DATA >> $TMT_PLAN_DATA/execution_order

tmt/base.py

@@ -3033,7 +3033,8 @@ def tests(
             unique: bool = True,
             links: Optional[list['LinkNeedle']] = None,
             excludes: Optional[list[str]] = None,
-            apply_command_line: bool = True
+            apply_command_line: bool = True,
+            sort: bool = True
             ) -> list[Test]:
         """ Search available tests """
         # Handle defaults, apply possible command line options
@@ -3076,8 +3077,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
 
         if Test._opt('source'):
             tests = [
-                Test(node=test, logger=self._logger.descend()) for test in self.tree.prune(
-                    keys=keys, sources=cmd_line_names)]
+                Test(node=test, logger=self._logger.descend())
+                for test in self.tree.prune(keys=keys, sources=cmd_line_names, sort=sort)]
 
         elif not unique and names:
             # First let's build the list of test objects based on keys & names.
@@ -3092,7 +3093,9 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
                         logger=logger.descend(
                             logger_name=test.get('name', None)
                             )  # .apply_verbosity_options(**self._options),
-                        ) for test in name_filter(self.tree.prune(keys=keys, names=[name]))]
+                        ) for test in name_filter(
+                            self.tree.prune(keys=keys, names=[name], sort=sort))
+                    ]
                 tests.extend(sorted(selected_tests, key=lambda test: test.order))
         # Otherwise just perform a regular key/name filtering
         else:
@@ -3103,7 +3106,8 @@ def name_filter(nodes: Iterable[fmf.Tree]) -> list[fmf.Tree]:
                     logger=logger.descend(
                         logger_name=test.get('name', None)
                         )  # .apply_verbosity_options(**self._options),
-                    ) for test in name_filter(self.tree.prune(keys=keys, names=names))]
+                    ) for test in name_filter(
+                        self.tree.prune(keys=keys, names=names, sort=sort))]
             tests = sorted(selected_tests, key=lambda test: test.order)
 
         # Apply filters & conditions

tmt/steps/discover/shell.py

@@ -438,7 +438,8 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
         self._tests = tmt.Tree(
             logger=self._logger,
             tree=tests).tests(
-                conditions=["manual is False"])
+                conditions=["manual is False"],
+                sort=False)
 
         # Propagate `where` key and TMT_SOURCE_DIR
         for test in self._tests: