Added allure suite, sub suite, parent suit, title

.allure-overview/overviews.json

@@ -15,6 +15,6 @@
 
     "performance": {
         "OVERVIEW_TITLE": "'OpenWifi performance results'",
-        "OVERVIEW_TESTBEDS": ["basic-01","basic-02","basic-03","basic-04","basic-05","basic-06","basic-07","basic-08","basic-08a"]
+        "OVERVIEW_TESTBEDS": ["wallys_dr40x9","hfcl_ion4","hfcl_ion4xe","edgecore_ecw5211","edgecore_eap101","edgecore_eap102","cig_wf194c","cig_wf188n","indio_um-305ac","udaya_a5-id2"]
     }
 }

.github/actions/run-tests/action.yml

@@ -129,14 +129,13 @@ runs:
         kubectl wait "pod/$podname" --for condition=ready --timeout=600s
 
         rm -f test_everything.xml
+        rm -rf allure-results
         until [ -s test_everything.xml ]
         do
           sleep 10
           kubectl cp $podname:/wlan-testing/tests/test_everything.xml test_everything.xml >/dev/null 2>&1
         done
         echo "tests completed"
-        kubectl logs $podname
-        rm -rf allure-results
         echo "downloading allure results..."
         kubectl cp $podname:/tmp/allure-results allure-results >/dev/null 2>&1
         echo "waiting for pod to exit"
@@ -151,9 +150,3 @@ runs:
       with:
         name: ${{ inputs.allure_results_artifact_name }}
         path: allure-results
-
-    - name: cleanup
-      if: always()
-      shell: bash
-      run: |
-        kubectl delete ns "${{ inputs.namespace }}" --wait=true

.github/workflows/advanced.yml

@@ -72,7 +72,14 @@ jobs:
         testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
         allure_results_artifact_name: allure-results-${{ matrix.testbed }}
 
-      # necessary because if conditionals in composite actions are currently not respected
+    # necessary because if conditionals in composite actions are currently not respected
+    - name: get tests logs
+      if: always()
+      continue-on-error: true
+      run: |
+        podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+        kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true
+
     - name: delete namespace
       if: always()
       continue-on-error: true

.github/workflows/interop.yml

@@ -16,11 +16,16 @@ on:
         description: comma-separated list of devices to test
         default: galaxy-s9,galaxy-s10,galaxy-s20,pixel-4,iphone-7,iphone-11,iphone-xr,iphone-12
         required: false
+      firmware:
+        default: "next-latest"
+        description: "target firmware version to be specified <branch>-<commit | latest>"
+        required: false
       marker_overwrite:
         description: overwrite the Pytest markers that will be selected, will disable report uploading
         required: false
   schedule:
-    - cron: "30 20 * * *"
+    - cron: "35 20 * * *"
+
 
 defaults:
   run:
@@ -32,9 +37,9 @@ jobs:
     outputs:
       devices: ${{ steps.vars.outputs.devices }}
       marker_overwrite: ${{ steps.vars.outputs.marker_overwrite }}
-      openwifi_revision: ${{ steps.vars.outputs.openwifi}}
-      ap_models: ${{ steps.vars.outputs.ap_models}}
-      ap_version: ${{ steps.vars.outputs.ap_version}}
+      openwifi_revision: ${{ steps.vars.outputs.openwifi }}
+      ap_models: ${{ steps.vars.outputs.ap_models }}
+      firmware: ${{ steps.vars.outputs.firmware }}
       marker_expression: ${{ steps.vars.outputs.marker_expression }}
       existing_controller: ${{ steps.vars.outputs.existing_controller }}
 
@@ -49,7 +54,7 @@ jobs:
           echo "::set-output name=devices::${DEVICES}"
           echo ::set-output name=openwifi::$(echo "${{ github.event.inputs.openwifi_revision || 'main' }}")
           echo ::set-output name=ap_models::$(echo "${{ github.event.inputs.ap_models || 'edgecore_ecw5410,edgecore_eap101,edgecore_ecw5211,cig_wf188n,edgecore_eap102,cig_wf194c,hfcl_ion4' }}")
-          echo ::set-output name=ap_version::$(echo "${{ github.event.inputs.ap_version || 'next-latest' }}")
+          echo ::set-output name=firmware::$(echo "${{ github.event.inputs.firmware || 'next-latest' }}")
           echo ::set-output name=marker_expression::$(echo "${{ github.event.inputs.marker_expression || 'uc_sanity' }}")
           echo ::set-output name=existing_controller::$(echo "${{ github.event.inputs.existing_controller || 'qa01' }}")
 
@@ -135,7 +140,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and android and not wpa3_personal' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='Galaxy S9' --skip-lanforge"
+          additional_args: "--device='Galaxy S9' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-galaxy-s9
 
       - name: stop reservation
@@ -147,6 +152,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-galaxy-s9 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-galaxy-s9 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -222,7 +234,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and ios' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='iPhone-11' --skip-lanforge"
+          additional_args: "--device='iPhone-11' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-iphone-11
 
       - name: stop reservation
@@ -234,6 +246,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-iphone-11 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-iphone-11 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -310,7 +329,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and android' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='Galaxy S10.*' --skip-lanforge"
+          additional_args: "--device='Galaxy S10.*' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-galaxy-s10
 
       - name: stop reservation
@@ -322,6 +341,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-galaxy-s10 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-galaxy-s10 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -399,7 +425,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and ios' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='iPhone-7' --skip-lanforge"
+          additional_args: "--device='iPhone-7' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-iphone-7
 
       - name: stop reservation
@@ -411,6 +437,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-iphone-7 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-iphone-7 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -486,7 +519,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and android' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='Galaxy S20' --skip-lanforge"
+          additional_args: "--device='Galaxy S20' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-galaxy-s20
 
       - name: stop reservation
@@ -498,6 +531,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-galaxy-s20 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-galaxy-s20 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -572,7 +612,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and ios' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='iPhone-XR' --skip-lanforge"
+          additional_args: "--device='iPhone-XR' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-iphone-xr
 
       - name: stop reservation
@@ -584,6 +624,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-iphone-xr --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-iphone-xr $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -660,7 +707,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and android' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='Pixel 4' --skip-lanforge"
+          additional_args: "--device='Pixel 4' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-pixel-4
 
       - name: stop reservation
@@ -672,6 +719,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-pixel-4 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-pixel-4 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true
@@ -745,7 +799,7 @@ jobs:
           marker_expression: "${{ needs.vars.outputs.marker_overwrite || 'ow_sanity_interop and ios' }}"
           configuration_file: "./lab_info.json"
           testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }}
-          additional_args: "--device='iPhone-12' --skip-lanforge"
+          additional_args: "--device='iPhone-12' --skip-lanforge -o firmware=${{ needs.vars.outputs.firmware }}"
           allure_results_artifact_name: allure-results-iphone-12
 
       - name: stop reservation
@@ -757,6 +811,13 @@ jobs:
           CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }}
 
       # necessary because if conditionals in composite actions are currently not respected
+      - name: get tests logs
+        if: always()
+        continue-on-error: true
+        run: |
+          podname=$(kubectl get pods -n interop-${{ github.run_id }}-iphone-12 --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///")
+          kubectl logs --timestamps -n interop-${{ github.run_id }}-iphone-12 $podname || true
+
       - name: delete namespace
         if: always()
         continue-on-error: true