Update report aligning to the mockup

daq/report.py

@@ -67,13 +67,17 @@ class ReportGenerator:
     _DEFAULT_EXPECTED = 'Other'
     _PRE_START_MARKER = "```"
     _PRE_END_MARKER = "```"
-    _CATEGORY_HEADERS = ["Category", "Result"]
+    _CATEGORY_BASE_COLS = ["Category", "Total Tests", "Result"]
     _EXPECTED_HEADER = "Expectation"
     _SUMMARY_HEADERS = ["Result", "Test", "Category", "Expectation", "Notes"]
     _MISSING_TEST_RESULT = 'gone'
     _NO_REQUIRED = 'n/a'
     _PASS_REQUIRED = 'PASS'
 
+    _INDEX_PASS = 0
+    _INDEX_FAIL = 1
+    _INDEX_SKIP = 2
+
     def __init__(self, config, tmp_base, target_mac, module_config):
         self._config = config
         self._module_config = copy.deepcopy(module_config)
@@ -101,7 +105,8 @@ def __init__(self, config, tmp_base, target_mac, module_config):
         self._expected_headers = list(self._module_config.get('report', {}).get('expected', []))
         self._expecteds = {}
         self._categories = list(self._module_config.get('report', {}).get('categories', []))
-
+        self._category_headers = []
+        self._append_notices = []
         self._file_md = None
 
     def _write(self, msg=''):
@@ -206,6 +211,7 @@ def _write_pdf_report(self):
 
     def _write_test_summary(self):
         self._writeln(self._TEST_SEPARATOR % self._SUMMARY_LINE)
+        self._analyse_and_write_results()
         self._write_test_tables()
 
     def _accumulate_result(self, test_name, result, extra='', module_name=None):
@@ -246,33 +252,102 @@ def _write_test_tables(self):
         self._write_result_table()
         self._writeln()
 
-    def _write_category_table(self):
+    def _analyse_and_write_results(self):
+        """ Analyse the test results to determine if the device is a pass or fail
+            and identify possible issues (e.g. gone) and writes these to the report
+        """
         passes = True
+        gone = False
+
+        # Analyse results
+        for test_name, result_dict in self._results.items():
+            test_info = self._get_test_info(test_name)
+
+            if 'required' in test_info:
+                required_result = test_info['required']
+
+                # The device overall fails if any result is unexpected
+                if result_dict["result"] != required_result:
+                    passes = False
+
+                if result_dict["result"] == 'gone':
+                    gone = True
+
+        # Write Results
+        self._writeln('Overall device result %s' % ('PASS' if passes else 'FAIL'))
+        self._writeln()
+
+        if gone:
+            gone_message = ('**Some tests report as GONE. '
+                            'Please check for possible misconfiguration**')
+            self._writeln(gone_message)
+            self._writeln()
+
+    def _join_category_results(self, results):
+        """ Used to convert list of results into the pass/fail/skip format
+            for category table
+
+        Args:
+        results: List of results
+
+        Returns:
+        String in pass/fail/skip format
+        """
+        return '/'.join(str(value) for value in results)
+
+    def _write_category_table(self):
+        """ Write the first category and expected table
+        """
+
         rows = []
+        self._category_headers = self._CATEGORY_BASE_COLS + self._expected_headers
+
         for category in self._categories:
             total = 0
-            match = 0
+
+            results = [[0, 0, 0] for _ in range(len(self._expected_headers))]
+            result = self._NO_REQUIRED  # Overall category result
+
             for test_name, result_dict in self._results.items():
                 test_info = self._get_test_info(test_name)
                 category_name = test_info.get('category', self._DEFAULT_CATEGORY)
+
+                # all tests must have required in order to be counted
                 if category_name == category and 'required' in test_info:
-                    required_result = test_info['required']
                     total += 1
-                    if result_dict["result"] == required_result:
-                        match += 1
+
+                    expected_name = test_info.get('expected', self._DEFAULT_EXPECTED)
+                    expected_index = self._expected_headers.index(expected_name)
+
+                    # Put test results into right location in the result matrix
+                    if result_dict["result"] in ("pass", "info"):
+                        results[expected_index][self._INDEX_PASS] += 1
+                    elif result_dict["result"] == "skip":
+                        results[expected_index][self._INDEX_SKIP] += 1
+                    else:
+                        results[expected_index][self._INDEX_FAIL] += 1
+
+                    # Calculate overall rolling result for the category
+                    if result not in (result_dict["result"], self._NO_REQUIRED):
+                        # Consider info and pass alike
+                        # TODO remove when info tests are removed
+                        if result_dict["result"] == 'info':
+                            result_dict["result"] = 'pass'
+                        else:
+                            result = "fail"
                     else:
-                        passes = False
+                        result = result_dict["result"]
 
-            output = self._NO_REQUIRED if total == 0 else (self._PASS_REQUIRED \
-                     if match == total else '%s/%s' % (match, total))
-            rows.append([category, output])
+            results = list(map(self._join_category_results, results))
 
-        self._writeln('Overall device result %s' % ('PASS' if passes else 'FAIL'))
-        self._writeln()
-        table = MdTable(self._CATEGORY_HEADERS)
+            row = [category, str(total), result.upper()] + results
+            rows.append(row)
+
+        table = MdTable(self._category_headers)
         for row in rows:
             table.add_row(row)
         self._write(table.render())
+        self._writeln('Syntax: Pass / Fail / Skip')
 
     def _write_expected_table(self):
         table = MdTable([self._EXPECTED_HEADER, *self._result_headers])

docs/device_report.md

@@ -46,49 +46,68 @@
 
 Overall device result FAIL
 
-|Category|Result|
-|---|---|
-|Security|1/2|
-|Other|1/2|
-|Connectivity|n/a|
+**Some tests report as GONE. Please check for possible misconfiguration**
+
+|Category|Total Tests|Result|Required Pass|Required Pass for PoE Devices|Required Pass for BACnet Devices|Recommended Pass|Information|Other|
+|---|---|---|---|---|---|---|---|---|
+|Connection|9|FAIL|1/4/4|0/0/0|0/0/0|0/0/0|0/0/0|0/0/0|
+|Security|8|FAIL|1/0/4|0/0/0|0/0/0|1/1/0|0/0/1|0/0/0|
+|Network Time|2|PASS|2/0/0|0/0/0|0/0/0|0/0/0|0/0/0|0/0/0|
+|TLS|0|N/A|0/0/0|0/0/0|0/0/0|0/0/0|0/0/0|0/0/0|
+|Protocol|2|FAIL|0/0/0|0/0/0|0/1/0|0/0/0|0/0/1|0/0/0|
+|PoE|3|FAIL|0/0/0|0/1/1|0/0/0|0/0/0|0/1/0|0/0/0|
+|BOS|1|SKIP|0/0/0|0/0/0|0/0/0|0/0/1|0/0/0|0/0/0|
+|Other|2|GONE|0/0/0|0/0/0|0/0/0|0/0/0|0/0/0|0/2/0|
+|Communication|2|GONE|0/1/0|0/0/0|0/0/0|0/0/0|0/1/0|0/0/0|
+Syntax: Pass / Fail / Skip
 
 |Expectation|pass|fail|skip|info|gone|
 |---|---|---|---|---|---|
-|Required|1|0|0|0|0|
-|Recommended|1|0|0|0|1|
-|Other|6|2|21|2|2|
+|Required Pass|4|1|8|0|4|
+|Required Pass for PoE Devices|0|0|1|0|1|
+|Required Pass for BACnet Devices|0|1|0|0|0|
+|Recommended Pass|1|0|1|0|1|
+|Information|0|0|2|0|2|
+|Other|3|0|9|2|2|
 
 |Result|Test|Category|Expectation|Notes|
 |---|---|---|---|---|
 |pass|base.startup.dhcp|Other|Other||
-|skip|base.switch.ping|Other|Other|No local IP has been set, check system config|
-|skip|cloud.udmi.pointset|Other|Other|No device id|
+|skip|base.switch.ping|Connection|Required Pass|No local IP has been set, check system config|
+|skip|cloud.udmi.pointset|BOS|Recommended Pass|No device id|
 |skip|cloud.udmi.provision|Other|Other|No device id|
 |skip|cloud.udmi.state|Other|Other|No device id|
 |skip|cloud.udmi.system|Other|Other|No device id|
 |pass|communication.network.min_send|Other|Other|ARP packets received. Data packets were sent at a frequency of less than 5 minutes|
 |info|communication.network.type|Other|Other|Broadcast packets received. Unicast packets received.|
-|pass|connection.base.target_ping|Connectivity|Required|target reached|
+|pass|connection.base.target_ping|Connection|Required Pass|target reached|
+|gone|connection.ipaddr.dhcp_disconnect|Connection|Required Pass||
+|gone|connection.ipaddr.private_address|Connection|Required Pass||
+|gone|connection.network.communication_min_send|Communication|Required Pass||
+|gone|connection.network.communication_type|Communication|Information||
+|gone|connection.network.dhcp_long|Connection|Required Pass||
 |info|connection.network.mac_address|Other|Other|Device MAC address is 9a:02:57:1e:8f:01|
-|fail|connection.network.mac_oui|Other|Other|Manufacturer prefix not found!|
-|skip|connection.switch.port_duplex|Other|Other|No local IP has been set, check system config|
-|skip|connection.switch.port_link|Other|Other|No local IP has been set, check system config|
-|skip|connection.switch.port_speed|Other|Other|No local IP has been set, check system config|
+|fail|connection.network.mac_oui|Connection|Required Pass|Manufacturer prefix not found!|
+|skip|connection.switch.port_duplex|Connection|Required Pass|No local IP has been set, check system config|
+|skip|connection.switch.port_link|Connection|Required Pass|No local IP has been set, check system config|
+|skip|connection.switch.port_speed|Connection|Required Pass|No local IP has been set, check system config|
 |skip|dns.network.hostname_resolution|Other|Other|Device did not send any DNS requests|
-|pass|manual.test.name|Security|Recommended|Manual test - for testing|
-|pass|ntp.network.ntp_support|Other|Other|Using NTPv4.|
-|pass|ntp.network.ntp_update|Other|Other|Device clock synchronized.|
-|skip|poe.switch.power|Other|Other|No local IP has been set, check system config|
-|fail|protocol.bacext.pic|Other|Other|PICS file defined however a BACnet device was not found.|
-|skip|protocol.bacext.version|Other|Other|Bacnet device not found.|
-|skip|security.discover.firmware|Other|Other|Could not retrieve a firmware version with nmap. Check bacnet port.|
+|pass|manual.test.name|Security|Recommended Pass|Manual test - for testing|
+|pass|ntp.network.ntp_support|Network Time|Required Pass|Using NTPv4.|
+|pass|ntp.network.ntp_update|Network Time|Required Pass|Device clock synchronized.|
+|gone|poe.switch.negotiation|PoE|Required Pass for PoE Devices||
+|skip|poe.switch.power|PoE|Required Pass for PoE Devices|No local IP has been set, check system config|
+|gone|poe.switch.support|PoE|Information||
+|fail|protocol.bacext.pic|Protocol|Required Pass for BACnet Devices|PICS file defined however a BACnet device was not found.|
+|skip|protocol.bacext.version|Protocol|Information|Bacnet device not found.|
+|skip|security.discover.firmware|Security|Information|Could not retrieve a firmware version with nmap. Check bacnet port.|
 |pass|security.nmap.http|Other|Other|No running http servers have been found.|
-|pass|security.nmap.ports|Other|Other|Only allowed ports found open.|
-|skip|security.password.http|Other|Other|Port 80 not open on target device.|
-|skip|security.password.https|Other|Other|Port 443 not open on target device.|
-|skip|security.password.ssh|Other|Other|Port 22 not open on target device.|
-|skip|security.password.telnet|Other|Other|Port 23 not open on target device.|
-|gone|security.ports.nmap|Security|Recommended||
+|pass|security.nmap.ports|Security|Required Pass|Only allowed ports found open.|
+|skip|security.password.http|Security|Required Pass|Port 80 not open on target device.|
+|skip|security.password.https|Security|Required Pass|Port 443 not open on target device.|
+|skip|security.password.ssh|Security|Required Pass|Port 22 not open on target device.|
+|skip|security.password.telnet|Security|Required Pass|Port 23 not open on target device.|
+|gone|security.ports.nmap|Security|Recommended Pass||
 |skip|security.tls.v1_2_client|Other|Other|No client initiated TLS communication detected|
 |skip|security.tls.v1_2_server|Other|Other|IOException unable to connect to server.|
 |skip|security.tls.v1_3_client|Other|Other|No client initiated TLS communication detected|

resources/setups/qualification/system_module_config.json

@@ -108,7 +108,7 @@
     "connection.network.communication_type": {
       "category": "Communication",
       "required": "info",
-      "expected": "information"
+      "expected": "Information"
     },
     "connection.network.communication_min_send": {
       "category": "Communication",