Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Record suite info in test summary yaml. #949

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
22 changes: 22 additions & 0 deletions mobly/records.py
Original file line number Diff line number Diff line change
@@ -94,6 +94,8 @@ class TestSummaryEntryType(enum.Enum):
SUMMARY = 'Summary'
# Information on the controllers used in a test class.
CONTROLLER_INFO = 'ControllerInfo'
# Test suite level information.
SUITE_INFO = 'SuiteInfo'
# Additional data added by users during test.
# This can be added at any point in the test, so do not assume the location
# of these entries in the summary file.
@@ -724,3 +726,23 @@ def summary_dict(self):
d['Skipped'] = len(self.skipped)
d['Error'] = len(self.error)
return d


class SuiteInfoRecord:
"""A record representing the suite info in test summary."""

KEY_SUITE_CLASS_NAME = 'Suite Class Name'
KEY_TIMESTAMP = 'Timestamp'

def __init__(self, suite_class_name):
self.suite_class_name = suite_class_name
self.timestamp = time.time()

def to_dict(self):
result = {}
result[self.KEY_SUITE_CLASS_NAME] = self.suite_class_name
result[self.KEY_TIMESTAMP] = self.timestamp
return result

def __repr__(self):
return str(self.to_dict())
15 changes: 14 additions & 1 deletion mobly/suite_runner.py
Original file line number Diff line number Diff line change
@@ -68,11 +68,13 @@ def setup_suite(self, config):
import collections
import inspect
import logging
import os
import sys

from mobly import base_test
from mobly import base_suite
from mobly import config_parser
from mobly import records
from mobly import signals
from mobly import test_runner

@@ -188,6 +190,16 @@ def _print_test_names(test_classes):
print(f'{cls.TAG}.{name}')


def _record_suite_info(log_path, suite_class_name):
"""Adds the record for suite information in test summary file."""
summary_path = os.path.join(log_path, records.OUTPUT_FILE_SUMMARY)
summary_writer = records.TestSummaryWriter(summary_path)
content = records.SuiteInfoRecord(suite_class_name=suite_class_name)
summary_writer.dump(
content.to_dict(), records.TestSummaryEntryType.SUITE_INFO
)


def run_suite_class(argv=None):
"""Executes tests in the test suite.

@@ -212,7 +224,8 @@ def run_suite_class(argv=None):
suite = suite_class(runner, config)
console_level = logging.DEBUG if cli_args.verbose else logging.INFO
ok = False
with runner.mobly_logger(console_level=console_level):
with runner.mobly_logger(console_level=console_level) as log_path:
_record_suite_info(log_path, suite_class_name=suite_class.__name__)
try:
suite.setup_suite(config.copy())
try:
11 changes: 11 additions & 0 deletions tests/mobly/records_test.py
Original file line number Diff line number Diff line change
@@ -542,6 +542,17 @@ def test_uid_helper():

self.assertEqual(test_uid_helper.uid, 'some-uuid')

def test_convert_suite_info_record_to_dict(self):
suite_class_name = 'FakeTestSuite'
record = records.SuiteInfoRecord(suite_class_name=suite_class_name)

result = record.to_dict()

self.assertIn(
(records.SuiteInfoRecord.KEY_SUITE_CLASS_NAME, suite_class_name),
result.items(),
)


if __name__ == '__main__':
unittest.main()
82 changes: 63 additions & 19 deletions tests/mobly/suite_runner_test.py
Original file line number Diff line number Diff line change
@@ -13,18 +13,22 @@
# limitations under the License.

import io
import logging
import os
import shutil
import sys
import tempfile
import time
import unittest
from unittest import mock

from mobly import base_suite
from mobly import base_test
from mobly import records
from mobly import suite_runner
from tests.lib import integration2_test
from tests.lib import integration_test
import yaml


class FakeTest1(base_test.BaseTestClass):
@@ -136,8 +140,9 @@ def test_run_suite_with_failures(self, mock_exit):
mock_exit.assert_called_once_with(1)

@mock.patch('sys.exit')
@mock.patch.object(suite_runner, '_find_suite_class', autospec=True)
def test_run_suite_class(self, mock_find_suite_class, mock_exit):
def test_run_suite_class(self, mock_exit):
tmp_file_path = self._gen_tmp_config_file()
mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
mock_called = mock.MagicMock()

class FakeTestSuite(base_suite.BaseSuite):
@@ -151,30 +156,55 @@ def teardown_suite(self):
mock_called.teardown_suite()
super().teardown_suite()

mock_find_suite_class.return_value = FakeTestSuite

tmp_file_path = os.path.join(self.tmp_dir, 'config.yml')
with io.open(tmp_file_path, 'w', encoding='utf-8') as f:
f.write(
"""
TestBeds:
# A test bed where adb will find Android devices.
- Name: SampleTestBed
Controllers:
MagicDevice: '*'
"""
)

mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
sys.modules['__main__'].__dict__[FakeTestSuite.__name__] = FakeTestSuite

with mock.patch.object(sys, 'argv', new=mock_cli_args):
suite_runner.run_suite_class()
try:
suite_runner.run_suite_class()
finally:
del sys.modules['__main__'].__dict__[FakeTestSuite.__name__]

mock_find_suite_class.assert_called_once()
mock_called.setup_suite.assert_called_once_with()
mock_called.teardown_suite.assert_called_once_with()
mock_exit.assert_not_called()

@mock.patch('sys.exit')
@mock.patch.object(time, 'time', return_value=1733143236.278318)
def test_run_suite_class_records_suite_class_name(self, mock_time, _):
tmp_file_path = self._gen_tmp_config_file()
mock_cli_args = ['test_binary', f'--config={tmp_file_path}']
expected_summary_entry = records.SuiteInfoRecord(
suite_class_name='FakeTestSuite'
).to_dict()
expected_summary_entry['Type'] = (
records.TestSummaryEntryType.SUITE_INFO.value
)

class FakeTestSuite(base_suite.BaseSuite):

def setup_suite(self, config):
super().setup_suite(config)
self.add_test_class(FakeTest1)

sys.modules['__main__'].__dict__[FakeTestSuite.__name__] = FakeTestSuite

with mock.patch.object(sys, 'argv', new=mock_cli_args):
try:
suite_runner.run_suite_class()
finally:
del sys.modules['__main__'].__dict__[FakeTestSuite.__name__]

summary_path = os.path.join(
logging.root_output_path, records.OUTPUT_FILE_SUMMARY
)
with io.open(summary_path, 'r', encoding='utf-8') as f:
summary_entries = list(yaml.safe_load_all(f))

self.assertIn(
expected_summary_entry,
summary_entries,
)

def test_print_test_names(self):
mock_test_class = mock.MagicMock()
mock_cls_instance = mock.MagicMock()
@@ -191,6 +221,20 @@ def test_print_test_names_with_exception(self):
mock_cls_instance._pre_run.side_effect = Exception('Something went wrong.')
mock_cls_instance._clean_up.assert_called_once()

def _gen_tmp_config_file(self):
tmp_file_path = os.path.join(self.tmp_dir, 'config.yml')
with io.open(tmp_file_path, 'w', encoding='utf-8') as f:
f.write(
"""
TestBeds:
# A test bed where adb will find Android devices.
- Name: SampleTestBed
Controllers:
MagicDevice: '*'
"""
)
return tmp_file_path


if __name__ == '__main__':
unittest.main()