diff --git a/.packit.yaml b/.packit.yaml index a0c88cb221..bdd854a89a 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -57,6 +57,23 @@ jobs: - fedora-all - epel-9 + # Test pull requests (provision) + - job: tests + identifier: provision + trigger: pull_request + manual_trigger: true + use_internal_tf: True + targets: + - fedora-latest-stable + tf_extra_params: + test: + tmt: + name: /plans/provision/virtual + environments: + - tmt: + context: + how: provision + # Test internal plugins - job: tests trigger: pull_request diff --git a/docs/contribute.rst b/docs/contribute.rst index d950091914..303e301af9 100644 --- a/docs/contribute.rst +++ b/docs/contribute.rst @@ -121,6 +121,15 @@ Run selected tests or plans in verbose mode: tmt run --verbose plan --name basic tmt run -v test -n smoke + +Full Test +------------------------------------------------------------------ + +.. warning:: + + This full test approach is being obsoleted. + See the :ref:`provision-methods` section for details. + Build the rpms and execute the whole test coverage, including tests which need the full virtualization support: @@ -162,6 +171,10 @@ Similar as above but run only tests which don't run for merge requests: cd tests/full make test-complement + +Unit Tests +------------------------------------------------------------------ + To run unit tests in hatch environment using pytest and generate coverage report: .. code-block:: shell @@ -197,12 +210,50 @@ versions. For example: fixtures, see https://docs.pytest.org/en/7.1.x/how-to/unittest.html for details. -.. note:: +.. _provision-methods: + +Provision Methods +------------------------------------------------------------------ + +Tests which exercise various provision methods should use the +``PROVISION_METHODS`` environment variable to select which +provision methods they can utilize during their execution. This +variable is likely to have ``local`` set as the default value to +execute directly on the test runner as the default scenario. + +The following tags can be used to enable given test under the +respective provision method plan: + +provision-artemis + For tests checking the :ref:`/spec/plans/provision/artemis` + plugin functionality. + +provision-beaker + For tests checking the :ref:`/spec/plans/provision/beaker` + plugin functionality using the ``mrack`` plugin. + +provision-connect + For tests checking the :ref:`/spec/plans/provision/connect` + plugin functionality. + +provision-container + For tests checking the :ref:`/spec/plans/provision/container` + provision method using the ``podman`` plugin. + +provision-virtual + For tests checking the :ref:`/spec/plans/provision/virtual` + provision method using the ``testcloud`` plugin. + +provision-ssh + Tests which are not tied to a specific provision method but + should be executed for all provision methods which are using + ``ssh`` to connect to guests. + +provision-only + Used to mark tests which are suitable to be run only under + specific provision methods. These will be excluded from + regular plans. - Tests which try various provision methods should use ``PROVISION_METHODS`` - environment variable to select which provision methods they can utilize - during their execution. This variable is likely to have default ``container`` - or ``local`` and use ``adjust`` rule for ``how=full`` to add ``virtual`` method. Docs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/plans/features/main.fmf b/plans/features/main.fmf index 5a63119e2a..fd45854730 100644 --- a/plans/features/main.fmf +++ b/plans/features/main.fmf @@ -1,21 +1,3 @@ -prepare+: - # Install jq and yq for cleaner tests - - how: install - package: - - jq - - python3-pip - - - how: shell - script: - # Try to install into .local first, if that fails, try without --user. - # In the sane world, tmt would be running in a virtualenv, and we'd - # install yq there. Otherwise, we'd change system packages, but we'd - # have to run as root to do that, and who's running tmt test suite - # as root? - # - - pip3 install --user yq || pip3 install yq - - yq --help - adjust+: - when: distro == centos-stream-8 and trigger == commit environment+: diff --git a/plans/features/steps.fmf b/plans/features/steps.fmf index 3747c5f88c..1189b69ed8 100644 --- a/plans/features/steps.fmf +++ b/plans/features/steps.fmf @@ -4,7 +4,7 @@ description: individual steps. discover: how: fmf - filter: "tier: 3" + filter: "tier: 3 & tag:-provision-only" /discover: discover+: diff --git a/plans/main.fmf b/plans/main.fmf index 4d837feb4a..6b2b03ae5f 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -2,6 +2,24 @@ provision: how: local +prepare+: + # Install jq and yq for cleaner tests + - how: install + package: + - jq + - python3-pip + + - how: shell + script: + # Try to install into .local first, if that fails, try without --user. + # In the sane world, tmt would be running in a virtualenv, and we'd + # install yq there. Otherwise, we'd change system packages, but we'd + # have to run as root to do that, and who's running tmt test suite + # as root? + # + - pip3 install --user yq || pip3 install yq + - yq --help + # Use the internal executor execute: how: tmt diff --git a/plans/provision/virtual.fmf b/plans/provision/virtual.fmf index 1e89d744be..77de744f8a 100644 --- a/plans/provision/virtual.fmf +++ b/plans/provision/virtual.fmf @@ -1,9 +1,47 @@ summary: Virtual machine via testcloud + +description: | + Verify functionality of the virtual provision plugin. Run + tests which exercise various features of the testcloud plugin + even if not directly related to the provision implementation. + + The plan takes care of fetching the image to speed up the + execution. Each test should set the `provision-virtual` tag to + get picked by the plan, consume the `PROVISION_METHODS` + variable and use `provision --how virtual` on their own. + discover: how: fmf - filter: 'tag:virtual & tier:1 | tag:provision-virtual-only' -provision: - how: virtual -# image: fedora -# memory: 2048 -# disk: 10 + filter: 'tag:provision-virtual' + +prepare+: + - name: required-packages + how: install + package: guestfs-tools + + - name: start-libvirtd + script: | + sudo systemctl start libvirtd + sudo systemctl status libvirtd + + - name: prepare-image + summary: Fetch the image, refresh dnf cache, install beakerlib + script: | + tmt run --remove plan --default provision --how virtual finish + for image in /var/tmp/tmt/testcloud/images/*qcow2; do + virt-customize --add $image --run-command 'dnf --refresh install -y beakerlib' + done + +environment: + PROVISION_METHODS: virtual + +adjust+: + - enabled: true + when: how == provision + + - provision: + hardware: + virtualization: + is-supported: true + memory: ">= 4 GB" + when: trigger == commit diff --git a/tests/core/escaping/main.fmf b/tests/core/escaping/main.fmf index 878f2e56c1..575391d3c2 100644 --- a/tests/core/escaping/main.fmf +++ b/tests/core/escaping/main.fmf @@ -6,11 +6,7 @@ description: test: ./test.sh framework: beakerlib tag-: [container, virtual] +tag+: [provision-virtual] environment: PROVISION_METHODS: local container -adjust: - - when: how == full - environment: - PROVISION_METHODS: container virtual local - tag+: [additional_coverage] diff --git a/tests/core/spaces/main.fmf b/tests/core/spaces/main.fmf index 1481b69cd4..fbf599a3a4 100644 --- a/tests/core/spaces/main.fmf +++ b/tests/core/spaces/main.fmf @@ -5,11 +5,7 @@ description: test: ./test.sh framework: beakerlib tag-: [container, virtual] +tag+: [provision-virtual] environment: PROVISION_METHODS: local container -adjust: - - when: how == full - environment: - PROVISION_METHODS: container virtual local - tag+: [additional_coverage] diff --git a/tests/discover/filtering.sh b/tests/discover/filtering.sh index 3fcd9d0324..368fc50a7c 100755 --- a/tests/discover/filtering.sh +++ b/tests/discover/filtering.sh @@ -125,20 +125,6 @@ rlJournalStart rlAssertGrep "path:" output rlPhaseEnd - # If plan or test weren't explicitly specified then fmf-ids for all tests - # in all plans should be shown - rlPhaseStartTest "fmf-id (w/o url): plans were executed if plan/test -n=." - ids_amount=$(tmt run -r discover -h fmf --fmf-id finish 2>&1 >/dev/null | - grep "name:" | - wc -l) - tests_amount=$(tmt run -r discover -h fmf finish 2>&1 >/dev/null | - grep "summary:" | - awk '{print $2}' | - awk '{s+=$1} END {print s}') - rlAssertEquals "Check that number of fmf-ids equals to tests number" \ - "$ids_amount" "$tests_amount" - rlPhaseEnd - # Checking the case when fmf_root = git_root rlPhaseStartTest "fmf-id (w/o url): path doesn't shown up" path="$(git rev-parse --show-toplevel)" diff --git a/tests/discover/libraries/main.fmf b/tests/discover/libraries/main.fmf index e61da270b5..91633c2931 100644 --- a/tests/discover/libraries/main.fmf +++ b/tests/discover/libraries/main.fmf @@ -13,8 +13,8 @@ require+: - tmt+provision-container - tmt+provision-virtual -adjust: - enabled: True +adjust+: + - enabled: True when: how == full because: this can be run only with a full virtualization tag+: [additional_coverage] diff --git a/tests/discover/main.fmf b/tests/discover/main.fmf index f6d8192751..ff4a07f886 100644 --- a/tests/discover/main.fmf +++ b/tests/discover/main.fmf @@ -38,7 +38,7 @@ tier: 3 summary: Get tests from the source tarball from dist-git lookaside test: ./distgit.sh enabled: false - adjust: + adjust+: - enabled: true when: how == full or trigger == commit tag+: [additional_coverage] diff --git a/tests/execute/duration/main.fmf b/tests/execute/duration/main.fmf index 495e555048..a5e309f0d5 100644 --- a/tests/execute/duration/main.fmf +++ b/tests/execute/duration/main.fmf @@ -2,11 +2,4 @@ summary: Verify that the test duration is correctly handled environment: PROVISION_METHODS: "local container" duration: 20m - -adjust: - when: how == full - environment: - PROVISION_METHODS: "local container virtual" - duration: 30m - because: with full virtualization more methods can be tested - tag+: [additional_coverage] +tag+: [provision-virtual] diff --git a/tests/execute/nonroot/main.fmf b/tests/execute/nonroot/main.fmf index e13b525073..3cd96ae265 100644 --- a/tests/execute/nonroot/main.fmf +++ b/tests/execute/nonroot/main.fmf @@ -2,13 +2,8 @@ summary: Scripts used in execute are pushed via sudo tag: - additional_coverage - virtual +- provision-virtual +- provision-only require: - tmt - tmt+provision-virtual -tier: "3" - -enabled: false -adjust: - - when: how == full - enabled: true - because: virtual provision needs full virtualization diff --git a/tests/execute/reboot/main.fmf b/tests/execute/reboot/main.fmf index 9ee3de01b9..f82cfb1c2f 100644 --- a/tests/execute/reboot/main.fmf +++ b/tests/execute/reboot/main.fmf @@ -8,15 +8,11 @@ tier: 4 /out-of-session: summary: Verify that reboot induced from outside of test session works test: ./out-of-session.sh + tag+: [provision-virtual] environment+: PROVISION_METHODS: container - adjust+: - - when: how==full - environment+: - PROVISION_METHODS: container virtual - # Disabled until the testing environment 'hardware specification' can be set. # https://tmt.readthedocs.io/en/stable/spec/plans.html#hardware /efi: @@ -35,8 +31,8 @@ tier: 4 test: ./shorten-timeout.sh enabled: false - adjust: - enabled: true + adjust+: + - enabled: true when: how == full because: this can be run only with full virtualization tag+: [additional_coverage] @@ -47,8 +43,8 @@ tier: 4 test: ./reuse.sh enabled: false - adjust: - enabled: true + adjust+: + - enabled: true when: how == full because: this can be run only with full virtualization tag+: [additional_coverage] diff --git a/tests/execute/rsync/main.fmf b/tests/execute/rsync/main.fmf index b316ed3756..d2428774ae 100644 --- a/tests/execute/rsync/main.fmf +++ b/tests/execute/rsync/main.fmf @@ -1,9 +1,5 @@ summary: Correctly handle rsync removed during test -enabled: false - -adjust: - enabled: true - duration: 10m - when: how == full - because: this can be run only with a full virtualization - tag+: [additional_coverage] +duration: 10m +tag: + - provision-virtual + - provision-only diff --git a/tests/execute/tty/main.fmf b/tests/execute/tty/main.fmf index 200960ec96..0f825fa533 100644 --- a/tests/execute/tty/main.fmf +++ b/tests/execute/tty/main.fmf @@ -1,9 +1,6 @@ summary: Verify TTY handling in test environment +tag+: [provision-virtual] +duration: 10m environment: PROVISION_METHODS: local container -adjust: - - when: how == full - environment: - PROVISION_METHODS: local container virtual - tag+: [additional_coverage] diff --git a/tests/execute/upgrade/main.fmf b/tests/execute/upgrade/main.fmf index cb154786dd..06d18552d3 100644 --- a/tests/execute/upgrade/main.fmf +++ b/tests/execute/upgrade/main.fmf @@ -8,10 +8,10 @@ tier: 4 duration: 1h # requires nested virtualization (runs in VM) enabled: false - adjust: - - enabled: true - when: how == full - tag+: [additional_coverage] + adjust+: + - enabled: true + when: how == full + tag+: [additional_coverage] /simple: summary: Run a single task from upgrades repo diff --git a/tests/finish/ansible/main.fmf b/tests/finish/ansible/main.fmf index 1ae5ba0a3d..9d091c7fe7 100644 --- a/tests/finish/ansible/main.fmf +++ b/tests/finish/ansible/main.fmf @@ -5,17 +5,13 @@ description: environment variables are correctly handed over and logs are pulled back. In the full testing mode check virtual and local provision method as well. +tag+: [provision-virtual] # By default we run under a container only environment: PROVISION_METHODS: container -adjust: +adjust+: - when: trigger == commit environment: PROVISION_METHODS: container local because: the pipeline does not support nested virtualization - - when: how == full - environment: - PROVISION_METHODS: container virtual local - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/login/main.fmf b/tests/login/main.fmf index 65431370c2..dac78ef1a9 100644 --- a/tests/login/main.fmf +++ b/tests/login/main.fmf @@ -21,10 +21,7 @@ /ready: summary: Try to log into guest to verify property is_ready test: ./ready.sh + duration: 15m + tag+: [provision-virtual] environment: PROVISION_METHODS: local container - adjust: - - when: how == full - environment: - PROVISION_METHODS: local container virtual - tag+: [additional_coverage] diff --git a/tests/prepare/ansible/main.fmf b/tests/prepare/ansible/main.fmf index 74dda5d623..2e0a37d70a 100644 --- a/tests/prepare/ansible/main.fmf +++ b/tests/prepare/ansible/main.fmf @@ -4,17 +4,13 @@ description: centos) and prepare it with an ansible playbook. Make sure the environment variables are correctly handed over. In the full testing mode check virtual and local provision method as well. +tag+: [provision-virtual] # By default we run under a container only environment: PROVISION_METHODS: container -adjust: +adjust+: - when: trigger == commit environment: PROVISION_METHODS: container local because: the pipeline does not support nested virtualization - - when: how == full - environment: - PROVISION_METHODS: container virtual local - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/prepare/install/main.fmf b/tests/prepare/install/main.fmf index a5c085d97a..b6504fb046 100644 --- a/tests/prepare/install/main.fmf +++ b/tests/prepare/install/main.fmf @@ -4,15 +4,11 @@ description: Make sure that special characters are correctly escaped. Verify that installation from copr works for epel7. Exercises all provision methods in full mode, container only by default. +tag+: [provision-virtual] environment: PROVISION_METHODS: container -adjust: +adjust+: - when: trigger == commit environment: PROVISION_METHODS: container local because: the pipeline does not support nested virtualization - - when: how == full - environment: - PROVISION_METHODS: container virtual local - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/prepare/multihost/main.fmf b/tests/prepare/multihost/main.fmf index ae32067cf4..147397d41b 100644 --- a/tests/prepare/multihost/main.fmf +++ b/tests/prepare/multihost/main.fmf @@ -1,16 +1,11 @@ summary: Check that requirement installation is guest-aware in multihost scenario +tag+: [provision-virtual] environment: PROVISION_METHODS: container -adjust: +adjust+: - when: trigger == commit environment: PROVISION_METHODS: container because: the pipeline does not support nested virtualization - - - when: how == full - environment: - PROVISION_METHODS: container virtual - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/prepare/recommend/main.fmf b/tests/prepare/recommend/main.fmf index c93452e96f..d223101224 100644 --- a/tests/prepare/recommend/main.fmf +++ b/tests/prepare/recommend/main.fmf @@ -3,15 +3,11 @@ description: Run a simple test which recommends two packages. Check that the existing one is installed and the other one is ignored. No error should be reported. +tag+: [provision-virtual] environment: PROVISION_METHODS: container -adjust: +adjust+: - when: trigger == commit environment: PROVISION_METHODS: container local because: the pipeline does not support nested virtualization - - when: how == full - environment: - PROVISION_METHODS: container virtual local - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/provision/reboot/main.fmf b/tests/provision/reboot/main.fmf index c850a5c7ba..fef9a20111 100644 --- a/tests/provision/reboot/main.fmf +++ b/tests/provision/reboot/main.fmf @@ -1,4 +1,5 @@ summary: Check that the reboot command works +tag+: [provision-virtual] environment: PROVISION_METHODS: container @@ -6,7 +7,7 @@ environment: adjust: - when: how == full environment: - PROVISION_METHODS: container connect virtual + PROVISION_METHODS: container connect tag+: [additional_coverage] - when: how == beaker diff --git a/tests/pull/results/main.fmf b/tests/pull/results/main.fmf index b693cf3a5d..91d72e75ce 100644 --- a/tests/pull/results/main.fmf +++ b/tests/pull/results/main.fmf @@ -3,11 +3,7 @@ description: Check that test logs are correctly fetched from the guest, test results are stored into `results.yaml` and plan data directory content is synced back. +tag+: [provision-virtual] environment: PROVISION_METHODS: container local -adjust: - - when: how == full - environment: - PROVISION_METHODS: container virtual local - tag+: [additional_coverage] diff --git a/tests/pull/simple/main.fmf b/tests/pull/simple/main.fmf index 2b0ef6855e..ef16704b4f 100644 --- a/tests/pull/simple/main.fmf +++ b/tests/pull/simple/main.fmf @@ -3,13 +3,9 @@ description: Make sure that provisioning a guest and clean it up works as expected even if no preparation, test execution or finish step scripts are performed. +tag+: [provision-virtual] link: - verifies: https://github.com/teemtee/tmt/issues/1011 environment: PROVISION_METHODS: container local -adjust: - - when: how == full - environment: - PROVISION_METHODS: container virtual local - tag+: [additional_coverage] diff --git a/tests/run/permissions/main.fmf b/tests/run/permissions/main.fmf index 353f944c12..4133dbd880 100644 --- a/tests/run/permissions/main.fmf +++ b/tests/run/permissions/main.fmf @@ -9,7 +9,7 @@ require+: - coreutils enabled: false -adjust: - enabled: true +adjust+: + - enabled: true when: how == full or trigger == commit because: it needs to be executed with root permissions diff --git a/tests/run/shell/main.fmf b/tests/run/shell/main.fmf index 3a09d3b6af..9f2d3578fa 100644 --- a/tests/run/shell/main.fmf +++ b/tests/run/shell/main.fmf @@ -8,7 +8,7 @@ tag+: - as_root - provision-virtual-only enabled: false -adjust: +adjust+: - enabled: true when: how == full tag+: [additional_coverage] diff --git a/tests/run/worktree/main.fmf b/tests/run/worktree/main.fmf index b728542518..6581a46200 100644 --- a/tests/run/worktree/main.fmf +++ b/tests/run/worktree/main.fmf @@ -12,13 +12,7 @@ require+: [tmt+provision-container] environment: PROVISION_METHODS: 'container' -adjust: +adjust+: - environment: PROVISION_METHODS: 'container local' when: trigger == commit - - environment: - PROVISION_METHODS: 'container local virtual' - result: xfail # issue 964 - duration: 10m - when: how == full - tag+: [additional_coverage] diff --git a/tests/security/ansible/main.fmf b/tests/security/ansible/main.fmf index c59b3fb851..ef3e5e427a 100644 --- a/tests/security/ansible/main.fmf +++ b/tests/security/ansible/main.fmf @@ -1,5 +1,6 @@ summary: Check whether an injected shell code is executed tier: 4 +tag+: [provision-virtual] # By default, we run under a container only environment: @@ -9,8 +10,3 @@ adjust: environment: PROVISION_METHODS: container local because: the pipeline does not support nested virtualization - - when: how == full - environment: - PROVISION_METHODS: container virtual local - because: local/virtual provision needs root/full virtualization - tag+: [additional_coverage] diff --git a/tests/steps/select/test.sh b/tests/steps/select/test.sh index 41ea46beee..2df71117a7 100755 --- a/tests/steps/select/test.sh +++ b/tests/steps/select/test.sh @@ -19,9 +19,9 @@ rlJournalStart rlRun "tmt run $options $selected_step 2>&1 >/dev/null | tee output" $exitcode for step in $steps; do if [[ $step == $selected_step ]]; then - rlAssertGrep $step output + rlAssertGrep "^ *$step" output else - rlAssertNotGrep $step output + rlAssertNotGrep "^ *$step" output fi done rlPhaseEnd diff --git a/tests/test/check/main.fmf b/tests/test/check/main.fmf index 446b435fb1..58c952c787 100644 --- a/tests/test/check/main.fmf +++ b/tests/test/check/main.fmf @@ -1,5 +1,6 @@ summary: Test test checks tier: 2 +tag+: [provision-virtual] /dmesg: test: ./test-dmesg.sh @@ -7,20 +8,8 @@ tier: 2 environment: PROVISION_METHODS: local container - adjust: - - when: how == full - environment: - PROVISION_METHODS: local container virtual - tag+: [additional_coverage] - /avc: test: ./test-avc.sh environment: PROVISION_METHODS: local - - adjust: - - when: how == full - environment: - PROVISION_METHODS: local virtual - tag+: [additional_coverage] diff --git a/tests/test/debug/main.fmf b/tests/test/debug/main.fmf index c5e9042dfe..bcfd98a2b6 100644 --- a/tests/test/debug/main.fmf +++ b/tests/test/debug/main.fmf @@ -2,11 +2,7 @@ summary: Verify that test debugging works fine description: Prepare the environment for testing using provision & prepare, then debug test code using repeated discover & execute. +tag+: [provision-virtual] environment: PROVISION_METHODS: local container -adjust: - environment: - PROVISION_METHODS: local container virtual - when: how == full - tag+: [additional_coverage] diff --git a/tmt/steps/finish/__init__.py b/tmt/steps/finish/__init__.py index 211c9df6fa..02f109257f 100644 --- a/tmt/steps/finish/__init__.py +++ b/tmt/steps/finish/__init__.py @@ -122,6 +122,11 @@ def go(self, force: bool = False) -> None: self.actions() return + # Nothing to do if no guests were provisioned + if not self.plan.provision.guests(): + self.warn("Nothing to finish, no guests provisioned.", shift=1) + return + # Prepare guests guest_copies: list[Guest] = [] diff --git a/tmt/steps/provision/testcloud.py b/tmt/steps/provision/testcloud.py index 71043d888f..bbfcef1f89 100644 --- a/tmt/steps/provision/testcloud.py +++ b/tmt/steps/provision/testcloud.py @@ -150,7 +150,7 @@ def import_testcloud() -> None: # VM defaults DEFAULT_BOOT_TIMEOUT = 120 # seconds -DEFAULT_CONNECT_TIMEOUT = 60 # seconds +DEFAULT_CONNECT_TIMEOUT = 120 # seconds NON_KVM_ADDITIONAL_WAIT = 20 # seconds NON_KVM_TIMEOUT_COEF = 10 # times