Record suite info in test summary yaml.

mobly/base_suite.py

@@ -95,3 +95,38 @@ def setup_suite(self, config):
   def teardown_suite(self):
     """Function used to add post tests cleanup tasks (optional)."""
     pass
+
+  # Optional interfaces that users can override to record customized suite
+  # information to test summary.
+
+  def get_suite_name(self):
+    """Override to return a customized suite name (optional).
+
+    Use suite class name by default.
+
+    Returns:
+      A string that indicates the suite name.
+    """
+    return self.__class__.__name__
+
+  def get_run_identifier(self):
+    """Override to record identifier describing the key run context (optional).
+
+    Users can include test runtime info as this method will be called after all
+    test classes are executed.
+
+    Returns:
+      A string that indicates key run context information.
+    """
+    return None
+
+  def get_suite_info(self):
+    """Override to record user defined extra info to test summary (optional).
+
+    Users can include test runtime info as this method will be called after all
+    test classes are executed.
+
+    Returns:
+      A dict of suite information. Keys and values must be serializable.
+    """
+    return {}

mobly/suite_runner.py

@@ -66,21 +66,97 @@ def setup_suite(self, config):
 """
 import argparse
 import collections
+import enum
 import inspect
 import logging
+import os
 import sys
 
 from mobly import base_test
 from mobly import base_suite
 from mobly import config_parser
+from mobly import records
 from mobly import signals
 from mobly import test_runner
+from mobly import utils
 
 
 class Error(Exception):
   pass
 
 
+class TestSummaryEntryType(enum.Enum):
+  """Constants used to record suite level entries in test summary file."""
+
+  SUITE_INFO = 'SuiteInfo'
+
+
+class SuiteInfoRecord:
+  """A record representing the test suite info in test summary.
+
+  This record class is for suites defined by inheriting `base_suite.BaseSuite`.
+  This is not for suites directly assembled via `run_suite`.
+  """
+
+  KEY_SUITE_NAME = 'Suite Name'
+  KEY_SUITE_CLASS_NAME = 'Suite Class Name'
+  KEY_RUN_IDENTIFIER = 'Run Identifier'
+  KEY_EXTRAS = 'Extras'
+  KEY_BEGIN_TIME = 'Suite Begin Time'
+  KEY_END_TIME = 'Suite End Time'
+
+  # The name of the test suite.
+  _suite_name: str
+  # The class name of the test suite class.
+  _suite_class_name: str
+  # The run identifier that describes key test run context.
+  _run_identifier: str | None = None
+  # User defined extra information of the test result. Must be serializable.
+  _extras: dict
+  # Epoch timestamp of when the suite started.
+  _begin_time: int | None = None
+  # Epoch timestamp of when the suite ended.
+  _end_time: int | None = None
+
+  def __init__(self, suite_class_name):
+    self._suite_class_name = suite_class_name
+    self._suite_name = ''
+    self._extras = dict()
+
+  def suite_begin(self):
+    """Call this when the suite begins execution."""
+    self._begin_time = utils.get_current_epoch_time()
+
+  def suite_end(self):
+    """Call this when the suite ends execution."""
+    self._end_time = utils.get_current_epoch_time()
+
+  def set_suite_name(self, suite_name):
+    """Sets the name of the test suite."""
+    self._suite_name = suite_name
+
+  def set_run_identifier(self, run_identifier):
+    """Sets the run identifier."""
+    self._run_identifier = run_identifier
+
+  def set_extras(self, extras):
+    """Sets extra information. Must be serializable."""
+    self._extras = extras
+
+  def to_dict(self):
+    result = {}
+    result[self.KEY_SUITE_CLASS_NAME] = self._suite_class_name
+    result[self.KEY_SUITE_NAME] = self._suite_name
+    result[self.KEY_RUN_IDENTIFIER] = self._run_identifier
+    result[self.KEY_EXTRAS] = self._extras
+    result[self.KEY_BEGIN_TIME] = self._begin_time
+    result[self.KEY_END_TIME] = self._end_time
+    return result
+
+  def __repr__(self):
+    return str(self.to_dict())
+
+
 def _parse_cli_args(argv):
   """Parses cli args that are consumed by Mobly.
 
@@ -230,6 +306,13 @@ def _print_test_names(test_classes):
       print(f'{cls.TAG}.{name}')
 
 
+def _dump_suite_info(suite_record, log_path):
+  """Dumps the suite info record to test summary file."""
+  summary_path = os.path.join(log_path, records.OUTPUT_FILE_SUMMARY)
+  summary_writer = records.TestSummaryWriter(summary_path)
+  summary_writer.dump(suite_record.to_dict(), TestSummaryEntryType.SUITE_INFO)
+
+
 def run_suite_class(argv=None):
   """Executes tests in the test suite.
 
@@ -254,19 +337,27 @@ def run_suite_class(argv=None):
   suite = suite_class(runner, config)
   test_selector = _parse_raw_test_selector(cli_args.tests)
   suite.set_test_selector(test_selector)
+  suite_record = SuiteInfoRecord(suite_class_name=suite_class.__name__)
+  suite_record.set_suite_name(suite.get_suite_name())
+
   console_level = logging.DEBUG if cli_args.verbose else logging.INFO
   ok = False
-  with runner.mobly_logger(console_level=console_level):
+  with runner.mobly_logger(console_level=console_level) as log_path:
     try:
       suite.setup_suite(config.copy())
       try:
+        suite_record.suite_begin()
         runner.run()
         ok = runner.results.is_all_pass
         print(ok)
       except signals.TestAbortAll:
         pass
     finally:
       suite.teardown_suite()
+      suite_record.suite_end()
+      suite_record.set_run_identifier(suite.get_run_identifier())
+      suite_record.set_extras(suite.get_suite_info())
+      _dump_suite_info(suite_record, log_path)
   if not ok:
     sys.exit(1)
 

tests/mobly/suite_runner_test.py

@@ -13,20 +13,25 @@
 # limitations under the License.
 
 import io
+import logging
 import os
 import shutil
 import sys
 import tempfile
+import time
 import unittest
 from unittest import mock
 
 from mobly import base_suite
 from mobly import base_test
+from mobly import records
 from mobly import suite_runner
 from mobly import test_runner
+from mobly import utils
 from tests.lib import integration2_test
 from tests.lib import integration_test
 from tests.lib import integration_test_suite
+import yaml
 
 
 class FakeTest1(base_test.BaseTestClass):
@@ -175,10 +180,11 @@ def teardown_suite(self):
     mock_called.set_test_selector.assert_called_once_with(None)
 
   @mock.patch('sys.exit')
+  @mock.patch.object(records, 'TestSummaryWriter', autospec=True)
   @mock.patch.object(suite_runner, '_find_suite_class', autospec=True)
   @mock.patch.object(test_runner, 'TestRunner')
   def test_run_suite_class_with_test_selection_by_class(
-      self, mock_test_runner_class, mock_find_suite_class, mock_exit
+      self, mock_test_runner_class, mock_find_suite_class, *_
   ):
     mock_test_runner = mock_test_runner_class.return_value
     mock_test_runner.results.is_all_pass = True
@@ -209,10 +215,11 @@ def setup_suite(self, config):
     )
 
   @mock.patch('sys.exit')
+  @mock.patch.object(records, 'TestSummaryWriter', autospec=True)
   @mock.patch.object(suite_runner, '_find_suite_class', autospec=True)
   @mock.patch.object(test_runner, 'TestRunner')
   def test_run_suite_class_with_test_selection_by_method(
-      self, mock_test_runner_class, mock_find_suite_class, mock_exit
+      self, mock_test_runner_class, mock_find_suite_class, *_
   ):
     mock_test_runner = mock_test_runner_class.return_value
     mock_test_runner.results.is_all_pass = True
@@ -243,12 +250,13 @@ def setup_suite(self, config):
     )
 
   @mock.patch('sys.exit')
+  @mock.patch.object(records, 'TestSummaryWriter', autospec=True)
   @mock.patch.object(test_runner, 'TestRunner')
   @mock.patch.object(
       integration_test_suite.IntegrationTestSuite, 'setup_suite', autospec=True
   )
   def test_run_suite_class_finds_suite_class_when_not_in_main_module(
-      self, mock_setup_suite, mock_test_runner_class, mock_exit
+      self, mock_setup_suite, mock_test_runner_class, *_
   ):
     mock_test_runner = mock_test_runner_class.return_value
     mock_test_runner.results.is_all_pass = True
@@ -260,6 +268,70 @@ def test_run_suite_class_finds_suite_class_when_not_in_main_module(
 
     mock_setup_suite.assert_called_once()
 
+  @mock.patch('sys.exit')
+  @mock.patch.object(
+      utils, 'get_current_epoch_time', return_value=1733143236278
+  )
+  def test_run_suite_class_records_suite_info(self, mock_time, _):
+    tmp_file_path = self._gen_tmp_config_file()
+    customized_suite_name = 'Customized Suite Name'
+    run_identifier = '123456'
+    mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
+    expected_record = suite_runner.SuiteInfoRecord(
+        suite_class_name='FakeTestSuite'
+    )
+    expected_record.set_suite_name(customized_suite_name)
+    expected_record.suite_begin()
+    expected_record.suite_end()
+    expected_record.set_run_identifier(run_identifier)
+    expected_record.set_extras(
+        {
+            'extra-key-0': 'extra-value-0',
+            'extra-key-1': 'extra-value-1',
+        }
+    )
+    expected_summary_entry = expected_record.to_dict()
+    expected_summary_entry['Type'] = (
+        suite_runner.TestSummaryEntryType.SUITE_INFO.value
+    )
+
+    class FakeTestSuite(base_suite.BaseSuite):
+
+      def get_suite_name(self):
+        return customized_suite_name
+
+      def get_run_identifier(self):
+        return run_identifier
+
+      def get_suite_info(self):
+        return {
+            'extra-key-0': 'extra-value-0',
+            'extra-key-1': 'extra-value-1',
+        }
+
+      def setup_suite(self, config):
+        super().setup_suite(config)
+        self.add_test_class(FakeTest1)
+
+    sys.modules['__main__'].__dict__[FakeTestSuite.__name__] = FakeTestSuite
+
+    with mock.patch.object(sys, 'argv', new=mock_cli_args):
+      try:
+        suite_runner.run_suite_class()
+      finally:
+        del sys.modules['__main__'].__dict__[FakeTestSuite.__name__]
+
+    summary_path = os.path.join(
+        logging.root_output_path, records.OUTPUT_FILE_SUMMARY
+    )
+    with io.open(summary_path, 'r', encoding='utf-8') as f:
+      summary_entries = list(yaml.safe_load_all(f))
+
+    self.assertIn(
+        expected_summary_entry,
+        summary_entries,
+    )
+
   def test_print_test_names(self):
     mock_test_class = mock.MagicMock()
     mock_cls_instance = mock.MagicMock()
@@ -276,6 +348,39 @@ def test_print_test_names_with_exception(self):
     mock_cls_instance._pre_run.side_effect = Exception('Something went wrong.')
     mock_cls_instance._clean_up.assert_called_once()
 
+  def test_convert_suite_info_record_to_dict(self):
+    suite_class_name = 'FakeTestSuite'
+    suite_name = 'Customized Suite Name'
+    run_identifier = '123456'
+    suite_version = '1.2.3'
+    record = suite_runner.SuiteInfoRecord(suite_class_name=suite_class_name)
+    record.set_extras({'version': suite_version})
+    record.set_suite_name(suite_name)
+    record.suite_begin()
+    record.suite_end()
+    record.set_run_identifier(run_identifier)
+
+    result = record.to_dict()
+
+    self.assertIn(
+        (suite_runner.SuiteInfoRecord.KEY_SUITE_CLASS_NAME, suite_class_name),
+        result.items(),
+    )
+    self.assertIn(
+        (suite_runner.SuiteInfoRecord.KEY_EXTRAS, {'version': suite_version}),
+        result.items(),
+    )
+    self.assertIn(
+        (suite_runner.SuiteInfoRecord.KEY_SUITE_NAME, suite_name),
+        result.items(),
+    )
+    self.assertIn(
+        (suite_runner.SuiteInfoRecord.KEY_RUN_IDENTIFIER, run_identifier),
+        result.items(),
+    )
+    self.assertIn(suite_runner.SuiteInfoRecord.KEY_BEGIN_TIME, result)
+    self.assertIn(suite_runner.SuiteInfoRecord.KEY_END_TIME, result)
+
   def _gen_tmp_config_file(self):
     tmp_file_path = os.path.join(self.tmp_dir, 'config.yml')
     with io.open(tmp_file_path, 'w', encoding='utf-8') as f: