Enable /plans/provision/virtual for pull requests

.packit.yaml

@@ -57,6 +57,23 @@ jobs:
       - fedora-all
       - epel-9
 
+  # Test pull requests (provision)
+  - job: tests
+    identifier: provision
+    trigger: pull_request
+    manual_trigger: true
+    use_internal_tf: True
+    targets:
+      - fedora-latest-stable
+    tf_extra_params:
+        test:
+            tmt:
+                name: /plans/provision/virtual
+        environments:
+          - tmt:
+                context:
+                    how: provision
+
   # Test internal plugins
   - job: tests
     trigger: pull_request

docs/contribute.rst

@@ -121,6 +121,15 @@ Run selected tests or plans in verbose mode:
     tmt run --verbose plan --name basic
     tmt run -v test -n smoke
 
+
+Full Test
+------------------------------------------------------------------
+
+.. warning::
+
+    This full test approach is being obsoleted.
+    See the :ref:`provision-methods` section for details.
+
 Build the rpms and execute the whole test coverage, including
 tests which need the full virtualization support:
 
@@ -162,6 +171,10 @@ Similar as above but run only tests which don't run for merge requests:
     cd tests/full
     make test-complement
 
+
+Unit Tests
+------------------------------------------------------------------
+
 To run unit tests in hatch environment using pytest and generate coverage report:
 
 .. code-block:: shell
@@ -197,12 +210,50 @@ versions. For example:
    fixtures, see https://docs.pytest.org/en/7.1.x/how-to/unittest.html for
    details.
 
-.. note::
+.. _provision-methods:
+
+Provision Methods
+------------------------------------------------------------------
+
+Tests which exercise various provision methods should use the
+``PROVISION_METHODS`` environment variable to select which
+provision methods they can utilize during their execution. This
+variable is likely to have ``local`` set as the default value to
+execute directly on the test runner as the default scenario.
+
+The following tags can be used to enable given test under the
+respective provision method plan:
+
+provision-artemis
+    For tests checking the :ref:`/spec/plans/provision/artemis`
+    plugin functionality.
+
+provision-beaker
+    For tests checking the :ref:`/spec/plans/provision/beaker`
+    plugin functionality using the ``mrack`` plugin.
+
+provision-connect
+    For tests checking the :ref:`/spec/plans/provision/connect`
+    plugin functionality.
+
+provision-container
+    For tests checking the :ref:`/spec/plans/provision/container`
+    provision method using the ``podman`` plugin.
+
+provision-virtual
+    For tests checking the :ref:`/spec/plans/provision/virtual`
+    provision method using the ``testcloud`` plugin.
+
+provision-ssh
+    Tests which are not tied to a specific provision method but
+    should be executed for all provision methods which are using
+    ``ssh`` to connect to guests.
+
+provision-only
+    Used to mark tests which are suitable to be run only under
+    specific provision methods. These will be excluded from
+    regular plans.
 
-   Tests which try various provision methods should use ``PROVISION_METHODS``
-   environment variable to select which provision methods they can utilize
-   during their execution. This variable is likely to have default ``container``
-   or ``local`` and use ``adjust`` rule for ``how=full`` to add ``virtual`` method.
 
 Docs
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

plans/features/main.fmf

@@ -1,21 +1,3 @@
-prepare+:
-  # Install jq and yq for cleaner tests
-  - how: install
-    package:
-      - jq
-      - python3-pip
-
-  - how: shell
-    script:
-      # Try to install into .local first, if that fails, try without --user.
-      # In the sane world, tmt would be running in a virtualenv, and we'd
-      # install yq there. Otherwise, we'd change system packages, but we'd
-      # have to run as root to do that, and who's running tmt test suite
-      # as root?
-      #
-      - pip3 install --user yq || pip3 install yq
-      - yq --help
-
 adjust+:
   - when: distro == centos-stream-8 and trigger == commit
     environment+:

plans/features/steps.fmf

@@ -4,7 +4,7 @@ description:
     individual steps.
 discover:
     how: fmf
-    filter: "tier: 3"
+    filter: "tier: 3 & tag:-provision-only"
 
 /discover:
     discover+:

plans/main.fmf

@@ -2,6 +2,24 @@
 provision:
     how: local
 
+prepare+:
+  # Install jq and yq for cleaner tests
+  - how: install
+    package:
+      - jq
+      - python3-pip
+
+  - how: shell
+    script:
+      # Try to install into .local first, if that fails, try without --user.
+      # In the sane world, tmt would be running in a virtualenv, and we'd
+      # install yq there. Otherwise, we'd change system packages, but we'd
+      # have to run as root to do that, and who's running tmt test suite
+      # as root?
+      #
+      - pip3 install --user yq || pip3 install yq
+      - yq --help
+
 # Use the internal executor
 execute:
     how: tmt

plans/provision/virtual.fmf

@@ -1,9 +1,47 @@
 summary: Virtual machine via testcloud
+
+description: |
+    Verify functionality of the virtual provision plugin. Run
+    tests which exercise various features of the testcloud plugin
+    even if not directly related to the provision implementation.
+
+    The plan takes care of fetching the image to speed up the
+    execution. Each test should set the `provision-virtual` tag to
+    get picked by the plan, consume the `PROVISION_METHODS`
+    variable and use `provision --how virtual` on their own.
+
 discover:
     how: fmf
-    filter: 'tag:virtual & tier:1 | tag:provision-virtual-only'
-provision:
-    how: virtual
-#    image: fedora
-#    memory: 2048
-#    disk: 10
+    filter: 'tag:provision-virtual'
+
+prepare+:
+  - name: required-packages
+    how: install
+    package: guestfs-tools
+
+  - name: start-libvirtd
+    script: |
+        sudo systemctl start libvirtd
+        sudo systemctl status libvirtd
+
+  - name: prepare-image
+    summary: Fetch the image, refresh dnf cache, install beakerlib
+    script: |
+        tmt run --remove plan --default provision --how virtual finish
+        for image in /var/tmp/tmt/testcloud/images/*qcow2; do
+            virt-customize --add $image --run-command 'dnf --refresh install -y beakerlib'
+        done
+
+environment:
+    PROVISION_METHODS: virtual
+
+adjust+:
+  - enabled: true
+    when: how == provision
+
+  - provision:
+        hardware:
+            virtualization:
+                is-supported: true
+            memory: ">= 4 GB"
+    when: trigger == commit

tests/core/escaping/main.fmf

@@ -6,11 +6,7 @@ description:
 test: ./test.sh
 framework: beakerlib
 tag-: [container, virtual]
+tag+: [provision-virtual]
 
 environment:
     PROVISION_METHODS: local container
-adjust:
-  - when: how == full
-    environment:
-        PROVISION_METHODS: container virtual local
-    tag+: [additional_coverage]

tests/core/spaces/main.fmf

@@ -5,11 +5,7 @@ description:
 test: ./test.sh
 framework: beakerlib
 tag-: [container, virtual]
+tag+: [provision-virtual]
 
 environment:
     PROVISION_METHODS: local container
-adjust:
-  - when: how == full
-    environment:
-        PROVISION_METHODS: container virtual local
-    tag+: [additional_coverage]

tests/discover/filtering.sh

@@ -125,20 +125,6 @@ rlJournalStart
         rlAssertGrep "path:" output
     rlPhaseEnd
 
-    # If plan or test weren't explicitly specified then fmf-ids for all tests
-    # in all plans should be shown
-    rlPhaseStartTest "fmf-id (w/o url): plans were executed if plan/test -n=."
-        ids_amount=$(tmt run -r discover -h fmf --fmf-id finish 2>&1 >/dev/null |
-                     grep "name:" |
-                     wc -l)
-        tests_amount=$(tmt run -r discover -h fmf finish 2>&1 >/dev/null |
-                       grep "summary:" |
-                       awk '{print $2}' |
-                       awk '{s+=$1} END {print s}')
-        rlAssertEquals "Check that number of fmf-ids equals to tests number" \
-                       "$ids_amount" "$tests_amount"
-    rlPhaseEnd
-
     # Checking the case when fmf_root = git_root
     rlPhaseStartTest "fmf-id (w/o url): path doesn't shown up"
         path="$(git rev-parse --show-toplevel)"

tests/discover/libraries/main.fmf

@@ -13,8 +13,8 @@ require+:
   - tmt+provision-container
   - tmt+provision-virtual
 
-adjust:
-    enabled: True
+adjust+:
+  - enabled: True
     when: how == full
     because: this can be run only with a full virtualization
     tag+: [additional_coverage]

tests/discover/main.fmf

@@ -38,7 +38,7 @@ tier: 3
     summary: Get tests from the source tarball from dist-git lookaside
     test: ./distgit.sh
     enabled: false
-    adjust:
+    adjust+:
     - enabled: true
       when: how == full or trigger == commit
       tag+: [additional_coverage]

tests/execute/duration/main.fmf

@@ -2,11 +2,4 @@ summary: Verify that the test duration is correctly handled
 environment:
     PROVISION_METHODS: "local container"
 duration: 20m
-
-adjust:
-    when: how == full
-    environment:
-        PROVISION_METHODS: "local container virtual"
-    duration: 30m
-    because: with full virtualization more methods can be tested
-    tag+: [additional_coverage]
+tag+: [provision-virtual]

tests/execute/nonroot/main.fmf

@@ -2,13 +2,8 @@ summary: Scripts used in execute are pushed via sudo
 tag:
 - additional_coverage
 - virtual
+- provision-virtual
+- provision-only
 require:
 - tmt
 - tmt+provision-virtual
-tier: "3"
-
-enabled: false
-adjust:
-  - when: how == full
-    enabled: true
-    because: virtual provision needs full virtualization

tests/execute/reboot/main.fmf

@@ -8,15 +8,11 @@ tier: 4
 /out-of-session:
     summary: Verify that reboot induced from outside of test session works
     test: ./out-of-session.sh
+    tag+: [provision-virtual]
 
     environment+:
         PROVISION_METHODS: container
 
-    adjust+:
-        - when: how==full
-          environment+:
-            PROVISION_METHODS: container virtual
-
 # Disabled until the testing environment 'hardware specification' can be set.
 # https://tmt.readthedocs.io/en/stable/spec/plans.html#hardware
 /efi:
@@ -35,8 +31,8 @@ tier: 4
     test: ./shorten-timeout.sh
     enabled: false
 
-    adjust:
-        enabled: true
+    adjust+:
+      - enabled: true
         when: how == full
         because: this can be run only with full virtualization
         tag+: [additional_coverage]
@@ -47,8 +43,8 @@ tier: 4
     test: ./reuse.sh
     enabled: false
 
-    adjust:
-        enabled: true
+    adjust+:
+      - enabled: true
         when: how == full
         because: this can be run only with full virtualization
         tag+: [additional_coverage]

tests/execute/rsync/main.fmf

@@ -1,9 +1,5 @@
 summary: Correctly handle rsync removed during test
-enabled: false
-
-adjust:
-  enabled: true
-  duration: 10m
-  when: how == full
-  because: this can be run only with a full virtualization
-  tag+: [additional_coverage]
+duration: 10m
+tag:
+  - provision-virtual
+  - provision-only

tests/execute/tty/main.fmf

@@ -1,9 +1,6 @@
 summary: Verify TTY handling in test environment
+tag+: [provision-virtual]
+duration: 10m
 
 environment:
     PROVISION_METHODS: local container
-adjust:
-  - when: how == full
-    environment:
-        PROVISION_METHODS: local container virtual
-    tag+: [additional_coverage]

tests/execute/upgrade/main.fmf

@@ -8,10 +8,10 @@ tier: 4
     duration: 1h
     # requires nested virtualization (runs in VM)
     enabled: false
-    adjust:
-    - enabled: true
-      when: how == full
-      tag+: [additional_coverage]
+    adjust+:
+      - enabled: true
+        when: how == full
+        tag+: [additional_coverage]
 
 /simple:
     summary: Run a single task from upgrades repo