diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.12__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.12__amd64-nightly.yaml index 787539ad97e64..d75aafa9cb2eb 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.12__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.12__amd64-nightly.yaml @@ -92,7 +92,6 @@ tests: steps: cluster_profile: alibabacloud-qe env: - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.13__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.13__amd64-nightly.yaml index 86018a083258b..0d0a6bd6a1a4c 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.13__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.13__amd64-nightly.yaml @@ -100,7 +100,6 @@ tests: steps: cluster_profile: alibabacloud-qe env: - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -315,7 +314,6 @@ tests: OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.13 env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -403,7 +401,6 @@ tests: cluster_profile: aws-qe env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-aws-ipi-proxy-cco-manual-security-token-service @@ -752,7 +749,6 @@ tests: cluster_profile: azure-qe env: BASE_DOMAIN: qe.azure.devcluster.openshift.com - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-azure-ipi-proxy-workers-rhcos-rhel8 @@ -973,7 +969,6 @@ tests: FIPS_MODE=true IP_STACK=v4 NETWORK_TYPE=OVNKubernetes - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-baremetalds-ipi-ovn @@ -1358,8 +1353,6 @@ tests: cron: 22 21 11,26 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-ipi-proxy-etcd-encryption @@ -1531,8 +1524,6 @@ tests: cron: 2 17 20 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-upi @@ -1645,8 +1636,6 @@ tests: cron: 3 13 9,24 * * steps: cluster_profile: ibmcloud-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-ibmcloud-ipi-workers-rhel8 @@ -1729,7 +1718,6 @@ tests: steps: cluster_profile: nutanix-qe-dis env: - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -1819,8 +1807,6 @@ tests: cron: 53 20 1,16 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-agent-vsphere-install-sno @@ -1880,8 +1866,6 @@ tests: cron: 35 18 6,21 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-vsphere-ipi-ovn-dualstack diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.14__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.14__amd64-nightly.yaml index 6726149c5e23d..a14e138d501d4 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.14__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.14__amd64-nightly.yaml @@ -117,7 +117,6 @@ tests: steps: cluster_profile: alibabacloud-qe env: - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" FIPS_ENABLED: "true" test: @@ -427,7 +426,6 @@ tests: OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.14 env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -817,7 +815,6 @@ tests: cluster_profile: aws-qe env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -1370,7 +1367,6 @@ tests: cluster_profile: azure-qe env: BASE_DOMAIN: qe.azure.devcluster.openshift.com - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-azure-ipi-proxy-workers-rhcos-rhel8 @@ -1707,7 +1703,6 @@ tests: FIPS_MODE=true IP_STACK=v4 NETWORK_TYPE=OVNKubernetes - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-baremetalds-ipi-ovn @@ -2156,8 +2151,6 @@ tests: cron: 16 19 29 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-ipi-proxy-etcd-encryption @@ -2251,8 +2244,6 @@ tests: cron: 8 6 2 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-upi @@ -2372,8 +2363,6 @@ tests: cron: 22 16 6,21 * * steps: cluster_profile: ibmcloud-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-ibmcloud-ipi-workers-rhel8 @@ -2486,7 +2475,6 @@ tests: steps: cluster_profile: nutanix-qe-dis env: - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" FIPS_ENABLED: "true" test: @@ -2600,8 +2588,6 @@ tests: cron: 33 15 15,30 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-agent-vsphere-install-sno @@ -2702,8 +2688,6 @@ tests: cron: 42 7 2 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-vsphere-ipi-ovn-dualstack diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__amd64-nightly.yaml index 38126c5a2479e..646ea94bd8a08 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.15__amd64-nightly.yaml @@ -117,7 +117,6 @@ tests: steps: cluster_profile: alibabacloud-qe env: - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" FIPS_ENABLED: "true" test: @@ -410,7 +409,6 @@ tests: OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.15 env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -697,7 +695,6 @@ tests: cluster_profile: aws-qe env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -1325,7 +1322,6 @@ tests: cluster_profile: azure-qe env: BASE_DOMAIN: qe.azure.devcluster.openshift.com - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-azure-ipi-proxy-workers-rhcos-rhel8 @@ -1651,7 +1647,6 @@ tests: FIPS_MODE=true IP_STACK=v4 NETWORK_TYPE=OVNKubernetes - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-baremetalds-ipi-ovn @@ -2014,8 +2009,6 @@ tests: cron: 23 19 4 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-ipi-proxy-etcd-encryption @@ -2109,8 +2102,6 @@ tests: cron: 24 22 15 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-upi @@ -2286,8 +2277,6 @@ tests: cron: 23 6 12,27 * * steps: cluster_profile: ibmcloud-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-ibmcloud-ipi-workers-rhel8 @@ -2400,7 +2389,6 @@ tests: steps: cluster_profile: nutanix-qe-dis env: - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" FIPS_ENABLED: "true" test: @@ -2508,8 +2496,6 @@ tests: cron: 14 7 7,22 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-agent-vsphere-install-sno @@ -2647,8 +2633,6 @@ tests: cron: 32 9 29 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-vsphere-ipi-ovn-dualstack diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml index 8d72f11ef89b4..311e8811de866 100644 --- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml +++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.16__amd64-nightly.yaml @@ -374,7 +374,6 @@ tests: OO_INDEX: quay.io/openshift-qe-optional-operators/aosqe-index:v4.16 env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' FIPS_ENABLED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -621,7 +620,6 @@ tests: cluster_profile: aws-qe env: BASE_DOMAIN: qe.devcluster.openshift.com - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" test: - chain: openshift-e2e-test-qe-ui @@ -1249,7 +1247,6 @@ tests: cluster_profile: azure-qe env: BASE_DOMAIN: qe.azure.devcluster.openshift.com - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-azure-ipi-proxy-workers-rhcos-rhel8 @@ -1458,7 +1455,6 @@ tests: FIPS_MODE=true IP_STACK=v4 NETWORK_TYPE=OVNKubernetes - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-baremetalds-ipi-ovn @@ -1804,8 +1800,6 @@ tests: cron: 21 4 5 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-ipi-proxy-etcd-encryption @@ -1899,8 +1893,6 @@ tests: cron: 27 7 7 * * steps: cluster_profile: gcp-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-gcp-upi @@ -2076,8 +2068,6 @@ tests: cron: 56 15 7,23 * * steps: cluster_profile: ibmcloud-qe - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-ibmcloud-ipi-workers-rhel8 @@ -2190,7 +2180,6 @@ tests: steps: cluster_profile: nutanix-qe-dis env: - E2E_RUN_TAGS: '@console' EXTRACT_MANIFEST_INCLUDED: "true" FIPS_ENABLED: "true" test: @@ -2298,8 +2287,6 @@ tests: cron: 32 15 11,27 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-agent-vsphere-install-sno @@ -2437,8 +2424,6 @@ tests: cron: 3 1 13 * * steps: cluster_profile: vsphere-2 - env: - E2E_RUN_TAGS: '@console' test: - chain: openshift-e2e-test-qe-ui workflow: cucushift-installer-rehearse-vsphere-ipi-ovn-dualstack diff --git a/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-commands.sh b/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-commands.sh index 8e220e3f46df2..1a394cb877fc0 100644 --- a/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-commands.sh +++ b/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-commands.sh @@ -216,8 +216,17 @@ function filter_tests() { echo_e2e_tags } -function test_execution() { - pushd verification-tests +function test_execution_cucumber() { + local test_type type_tags + test_type="$1" + type_tags="$2" + export BUSHSLICER_REPORT_DIR="${ARTIFACT_DIR}/${test_type}" + export OPENSHIFT_ENV_OCP4_USER_MANAGER_USERS="${USERS}" + set -x + cucumber --tags "${E2E_RUN_TAGS} and ${type_tags}" -p junit || true + set +x +} +function test_execution_default() { # run normal tests in parallel export BUSHSLICER_REPORT_DIR="${ARTIFACT_DIR}/parallel-normal" SECONDS=0 @@ -237,13 +246,45 @@ function test_execution() { show_test_execution_time 'admin' # run the rest tests in serial - export BUSHSLICER_REPORT_DIR="${ARTIFACT_DIR}/serial" - export OPENSHIFT_ENV_OCP4_USER_MANAGER_USERS="${USERS}" SECONDS=0 - set -x - cucumber --tags "${E2E_RUN_TAGS} and ((@console and @smoke) or @serial)" -p junit || true - set +x + test_execution_cucumber 'serial' '((@console and @smoke) or @serial)' show_test_execution_time 'smoke console or serial' +} +function test_execution_destructive() { + test_execution_cucumber 'destructive1' 'not @console' + test_execution_cucumber 'destructive2' '@console and @smoke' +} +function test_execution_longduration() { + test_execution_cucumber 'longduration1' 'not @destructive' + test_execution_cucumber 'longduration2' '@destructive' +} +function test_execution_ui() { + test_execution_cucumber 'ui1' 'not @destructive' + test_execution_cucumber 'ui2' '@destructive' +} +function test_execution() { + pushd verification-tests + case "$E2E_TEST_TYPE" in + default) + export E2E_RUN_TAGS="${E2E_RUN_TAGS} and not @destructive and not @long-duration" + test_execution_default + ;; + destructive) + export E2E_RUN_TAGS="${E2E_RUN_TAGS} and @destructive" + test_execution_destructive + ;; + longduration) + export E2E_RUN_TAGS="${E2E_RUN_TAGS} and @long-duration" + test_execution_longduration + ;; + ui) + export E2E_RUN_TAGS="${E2E_RUN_TAGS} and @console" + test_execution_ui + ;; + *) + echo "Got unexpected test type: $E2E_TEST_TYPE" + ;; + esac popd } function summarize_test_results() { @@ -277,10 +318,8 @@ EOF PARALLEL_CUCUMBER_OPTIONS='--verbose-process-command --first-is-1 --type cucumber --serialize-stdout --combine-stderr --prefix-output-with-test-env-number' CUCUSHIFT_FORCE_SKIP_TAGS="not @customer - and not @destructive and not @flaky and not @inactive - and not @long-duration and not @prod-only and not @qeci and not @security diff --git a/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-ref.yaml b/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-ref.yaml index 423c495bcc60b..36507180a5dcc 100644 --- a/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-ref.yaml +++ b/ci-operator/step-registry/cucushift/e2e/cucushift-e2e-ref.yaml @@ -21,6 +21,15 @@ ref: - name: E2E_RUN_TAGS default: "" documentation: "Executed cucumber tags" + - name: E2E_TEST_TYPE + default: "default" + documentation: |- + The kind of test we will execute. + Allowed values ['default', 'destructive', 'longduration', 'ui'] + * 'default' - All the test cases, exclude destructive/longduration + * 'destructive' - Test cases with destructive tag + * 'longduration' - Test cases with long-duration tag + * 'ui' - Test cases with console tag - name: PARALLEL default: "11" documentation: The number of parallel process to run cucushift tests diff --git a/ci-operator/step-registry/cucushift/e2e/ui/OWNERS b/ci-operator/step-registry/cucushift/e2e/ui/OWNERS deleted file mode 100644 index 140a70b4fff8f..0000000000000 --- a/ci-operator/step-registry/cucushift/e2e/ui/OWNERS +++ /dev/null @@ -1,9 +0,0 @@ -approvers: -- jhou1 -- liangxia -reviewers: -- jhou1 -- liangxia -- JianLi-RH -- dis016 -- pruan-rht diff --git a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-commands.sh b/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-commands.sh deleted file mode 100644 index 7c06cd05a0033..0000000000000 --- a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-commands.sh +++ /dev/null @@ -1,273 +0,0 @@ -#!/bin/bash - -set -o nounset -set -o errexit -set -o pipefail - -function set_cluster_access() { - if [ -f "${SHARED_DIR}/kubeconfig" ] ; then - export KUBECONFIG=${SHARED_DIR}/kubeconfig - echo "KUBECONFIG: ${KUBECONFIG}" - fi - cp -Lrvf "${KUBECONFIG}" /tmp/kubeconfig - if [ -f "${SHARED_DIR}/proxy-conf.sh" ] ; then - source "${SHARED_DIR}/proxy-conf.sh" - echo "proxy: ${SHARED_DIR}/proxy-conf.sh" - fi -} -function preparation_for_test() { - if ! which kubectl > /dev/null ; then - mkdir --parents /tmp/bin - export PATH=$PATH:/tmp/bin - ln --symbolic "$(which oc)" /tmp/bin/kubectl - fi - #shellcheck source=${SHARED_DIR}/runtime_env - source "${SHARED_DIR}/runtime_env" -} -function echo_e2e_tags() { - echo "In function: ${FUNCNAME[1]}" - echo "E2E_RUN_TAGS: '${E2E_RUN_TAGS}'" -} -function filter_test_by_version() { - local xversion yversion - IFS='.' read xversion yversion _ < <(oc version -o yaml | yq '.openshiftVersion') - if [[ -n $xversion ]] && [[ $xversion -eq 4 ]] && [[ -n $yversion ]] && [[ $yversion =~ [12][0-9] ]] ; then - export E2E_RUN_TAGS="@${xversion}.${yversion} and ${E2E_RUN_TAGS}" - fi - echo_e2e_tags -} -function filter_test_by_arch() { - local node_archs arch_tags - mapfile -t node_archs < <(oc get nodes -o yaml | yq '.items[].status.nodeInfo.architecture' | sort -u | sed 's/^/@/g') - arch_tags="${node_archs[*]/%/ and}" - case "${#node_archs[@]}" in - 0) - echo "==========================" - echo "Error: got unexpected arch" - oc get nodes -o yaml - echo "==========================" - ;; - 1) - export E2E_RUN_TAGS="${arch_tags[*]} ${E2E_RUN_TAGS}" - ;; - *) - export E2E_RUN_TAGS="@heterogeneous and ${arch_tags[*]} ${E2E_RUN_TAGS}" - ;; - esac - echo_e2e_tags -} -function filter_test_by_platform() { - local platform ipixupi - ipixupi='upi' - if (oc get configmap openshift-install -n openshift-config &>/dev/null) ; then - ipixupi='ipi' - fi - platform="$(oc get infrastructure cluster -o yaml | yq '.status.platform' | tr 'A-Z' 'a-z')" - extrainfoCmd="oc get infrastructure cluster -o yaml | yq '.status'" - if [[ -n "$platform" ]] ; then - case "$platform" in - none) - export E2E_RUN_TAGS="@baremetal-upi and ${E2E_RUN_TAGS}" - eval "$extrainfoCmd" - ;; - external) - echo "Expected, got platform as '$platform'" - eval "$extrainfoCmd" - ;; - alibabacloud) - export E2E_RUN_TAGS="@alicloud-${ipixupi} and ${E2E_RUN_TAGS}" - ;; - aws|azure|baremetal|gcp|ibmcloud|nutanix|openstack|vsphere) - export E2E_RUN_TAGS="@${platform}-${ipixupi} and ${E2E_RUN_TAGS}" - ;; - *) - echo "Unexpected, got platform as '$platform'" - eval "$extrainfoCmd" - ;; - esac - fi - echo_e2e_tags -} -function filter_test_by_network() { - local networktype - networktype="$(oc get network.config/cluster -o yaml | yq '.spec.networkType')" - case "${networktype,,}" in - openshiftsdn) - networktag='@network-openshiftsdn' - ;; - ovnkubernetes) - networktag='@network-ovnkubernetes' - ;; - *) - echo "######Expected network to be SDN/OVN, but got: $networktype" - ;; - esac - if [[ -n $networktag ]] ; then - export E2E_RUN_TAGS="${networktag} and ${E2E_RUN_TAGS}" - fi - echo_e2e_tags -} -function filter_test_by_sno() { - local nodeno - nodeno="$(oc get nodes --no-headers | wc -l)" - if [[ $nodeno -eq 1 ]] ; then - export E2E_RUN_TAGS="@singlenode and ${E2E_RUN_TAGS}" - fi - echo_e2e_tags -} -function filter_test_by_proxy() { - local proxy - proxy="$(oc get proxies.config.openshift.io cluster -o yaml | yq '.spec|(.httpProxy,.httpsProxy)' | uniq)" - if [[ -n "$proxy" ]] && [[ "$proxy" != 'null' ]] ; then - export E2E_RUN_TAGS="@proxy and ${E2E_RUN_TAGS}" - fi - echo_e2e_tags -} -function filter_test_by_fips() { - local data - data="$(oc get configmap cluster-config-v1 -n kube-system -o yaml | yq '.data')" - if ! (grep --ignore-case --quiet 'fips' <<< "$data") ; then - export E2E_RUN_TAGS="not @fips and ${E2E_RUN_TAGS}" - fi - echo_e2e_tags -} -function filter_test_by_capability() { - local enabledcaps xversion yversion - enabledcaps="$(oc get clusterversion version -o yaml | yq '.status.capabilities.enabledCapabilities[]')" - IFS='.' read xversion yversion _ < <(oc version -o yaml | yq '.openshiftVersion') - local v411 v412 v413 v414 v415 v416 - v411="baremetal marketplace openshift-samples" - v412="${v411} Console Insights Storage CSISnapshot" - v413="${v412} NodeTuning" - v414="${v413} MachineAPI Build DeploymentConfig ImageRegistry" - v415="${v414} OperatorLifecycleManager CloudCredential" - v416="${v415}" - # [console]=console - # the first `console` is the capability name - # the second `console` is the tag name in verification-tests - declare -A tagmaps - tagmaps=([baremetal]=xxx - [Build]=xxx - [CloudCredential]=xxx - [Console]=console - [CSISnapshot]=storage - [DeploymentConfig]=xxx - [ImageRegistry]=xxx - [Insights]=xxx - [MachineAPI]=xxx - [marketplace]=xxx - [NodeTuning]=xxx - [openshift-samples]=xxx - [OperatorLifecycleManager]=xxx - [Storage]=storage - ) - local versioncaps - versioncaps="$v416" - case "$xversion.$yversion" in - 4.16) - versioncaps="$v416" - ;; - 4.15) - versioncaps="$v415" - ;; - 4.14) - versioncaps="$v414" - ;; - 4.13) - versioncaps="$v413" - ;; - 4.12) - versioncaps="$v412" - ;; - 4.11) - versioncaps="$v411" - ;; - *) - versioncaps="" - echo "Got unexpected version: $xversion.$yversion" - ;; - esac - for cap in ${versioncaps} ; do - if ! (grep --ignore-case --quiet "$cap" <<< "$enabledcaps") ; then - if [[ "${tagmaps[$cap]}" != 'xxx' ]] ; then - export E2E_RUN_TAGS="not @${tagmaps[$cap]} and ${E2E_RUN_TAGS}" - else - echo "TO_BE_DONE: find tag map for '$cap'" - fi - fi - done - echo_e2e_tags -} -function filter_tests() { - filter_test_by_capability - filter_test_by_fips - filter_test_by_proxy - filter_test_by_sno - filter_test_by_network - filter_test_by_platform - filter_test_by_arch - filter_test_by_version - - echo_e2e_tags -} -function test_execution() { - pushd verification-tests - # run destructive tests in serial - export BUSHSLICER_REPORT_DIR="${ARTIFACT_DIR}/ui" - export OPENSHIFT_ENV_OCP4_USER_MANAGER_USERS="${USERS}" - set -x - cucumber --tags "${E2E_RUN_TAGS} and @console and not @destructive" -p junit || true - cucumber --tags "${E2E_RUN_TAGS} and @console and @destructive" -p junit || true - set +x - popd -} -function summarize_test_results() { - # summarize test results - echo "Summarizing test results..." - failures=0 errors=0 skipped=0 tests=0 - grep -r -E -h -o 'testsuite.*tests="[0-9]+"' "${ARTIFACT_DIR}" | tr -d '[A-Za-z=\"_]' > /tmp/zzz-tmp.log - while read -a row ; do - # if the last ARG of command `let` evaluates to 0, `let` returns 1 - let failures+=${row[0]} errors+=${row[1]} skipped+=${row[2]} tests+=${row[3]} || true - done < /tmp/zzz-tmp.log - TEST_RESULT_FILE="${ARTIFACT_DIR}/test-results.yaml" - cat > "${TEST_RESULT_FILE}" <<- EOF -cucushift: - type: cucushift-e2e-ui - total: $tests - failures: $failures - errors: $errors - skipped: $skipped -EOF - if [ $((failures)) != 0 ] ; then - echo ' failingScenarios:' >> "${TEST_RESULT_FILE}" - readarray -t failingscenarios < <(grep -h -r -E 'cucumber.*features/.*.feature' "${ARTIFACT_DIR}/.." | cut -d':' -f3- | sed -E 's/^( +)//;s/\x1b\[[0-9;]*m$//' | sort) - for (( i=0; i> "${TEST_RESULT_FILE}" - done - fi - cat "${TEST_RESULT_FILE}" | tee -a "${SHARED_DIR}/openshift-e2e-test-qe-report" || true -} - - -CUCUSHIFT_FORCE_SKIP_TAGS="not @customer - and not @flaky - and not @inactive - and not @long-duration - and not @prod-only - and not @qeci - and not @security - and not @stage-only - and not @upgrade-check - and not @upgrade-prepare -" -if [[ -z "$E2E_RUN_TAGS" ]] ; then - export E2E_RUN_TAGS="$CUCUSHIFT_FORCE_SKIP_TAGS" -else - export E2E_RUN_TAGS="$E2E_RUN_TAGS and $CUCUSHIFT_FORCE_SKIP_TAGS" -fi -set_cluster_access -preparation_for_test -filter_tests -test_execution -summarize_test_results diff --git a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.metadata.json b/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.metadata.json deleted file mode 100644 index a4fb1454c6afa..0000000000000 --- a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.metadata.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "path": "cucushift/e2e/ui/cucushift-e2e-ui-ref.yaml", - "owners": { - "approvers": [ - "jhou1", - "liangxia" - ], - "reviewers": [ - "jhou1", - "liangxia", - "JianLi-RH", - "dis016", - "pruan-rht" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.yaml b/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.yaml deleted file mode 100644 index eacaa083a0810..0000000000000 --- a/ci-operator/step-registry/cucushift/e2e/ui/cucushift-e2e-ui-ref.yaml +++ /dev/null @@ -1,23 +0,0 @@ -ref: - as: cucushift-e2e-ui - from_image: - namespace: ci - name: verification-tests - tag: latest - grace_period: 10m - timeout: 10h - commands: cucushift-e2e-ui-commands.sh - cli: latest - resources: - limits: - ci-operator.openshift.io/shm: 2G - cpu: "2" - memory: 6Gi - requests: - ci-operator.openshift.io/shm: 2G - cpu: 800m - memory: 1Gi - env: - - name: E2E_RUN_TAGS - default: "" - documentation: "Executed cucumber tags" diff --git a/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-commands.sh b/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-commands.sh index f8ed89a380c1e..315b6def9fd51 100644 --- a/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-commands.sh +++ b/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-commands.sh @@ -25,9 +25,9 @@ else if [[ $E2E_RUN_TAGS =~ @osd_ccs|@rosa ]] ; then echo "Testing against online cluster" ./console-test-managed-service.sh || true - # if the TAGS contains @console, then it's a job specific for UI, run full tests + # if the TYPE is ui, then it's a job specific for UI, run full tests # or else, we run smoke tests to balance coverage and cost - elif [[ $E2E_RUN_TAGS =~ @console ]]; then + elif [[ "X${E2E_TEST_TYPE}X" == 'XuiX' ]]; then echo "Testing on normal cluster" ./console-test-frontend.sh || true else diff --git a/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-ref.yaml b/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-ref.yaml index 007e7ddfd666c..f338bcc141d8d 100644 --- a/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-ref.yaml +++ b/ci-operator/step-registry/openshift-extended/web-tests/openshift-extended-web-tests-ref.yaml @@ -12,6 +12,9 @@ ref: cpu: "8" memory: 4Gi env: + - name: E2E_TEST_TYPE + default: "" + documentation: "The kind of test we will execute" - name: E2E_RUN_TAGS default: "" documentation: "Executed cucumber tags" diff --git a/ci-operator/step-registry/openshift/e2e/test/qe/ui/openshift-e2e-test-qe-ui-chain.yaml b/ci-operator/step-registry/openshift/e2e/test/qe/ui/openshift-e2e-test-qe-ui-chain.yaml index 222b94c32b1f0..b60e36722082a 100644 --- a/ci-operator/step-registry/openshift/e2e/test/qe/ui/openshift-e2e-test-qe-ui-chain.yaml +++ b/ci-operator/step-registry/openshift/e2e/test/qe/ui/openshift-e2e-test-qe-ui-chain.yaml @@ -1,10 +1,20 @@ chain: as: openshift-e2e-test-qe-ui + env: + - name: E2E_TEST_TYPE + default: "ui" + documentation: |- + The kind of test we will execute. + Allowed values ['default', 'destructive', 'longduration', 'ui'] + * 'default' - All the test cases, exclude destructive/longduration + * 'destructive' - Test cases with destructive tag + * 'longduration' - Test cases with long-duration tag + * 'ui' - Test cases with console tag steps: - chain: cucushift-installer-check-cluster-health - ref: idp-htpasswd - ref: cucushift-pre - - ref: cucushift-e2e-ui + - ref: cucushift-e2e - ref: openshift-extended-web-tests - ref: openshift-e2e-test-qe-report documentation: |-