From c18a5a99608cd5e7df0c697cdf60340642b223b8 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Tue, 19 Sep 2023 20:34:28 +0530 Subject: [PATCH] add utils/tests/test_scanner.py and test_run_unit_test in test_remote.py Signed-off-by: Vallari Agrawal --- teuthology/orchestra/test/test_remote.py | 19 ++ .../test/xml_files/test_scan_nose.xml | 73 ------- teuthology/util/test/files/test_unit_test.xml | 7 + teuthology/util/test/files/test_valgrind.xml | 31 +++ teuthology/util/test/test_scanner.py | 191 ++++++++++++++++++ 5 files changed, 248 insertions(+), 73 deletions(-) delete mode 100644 teuthology/orchestra/test/xml_files/test_scan_nose.xml create mode 100644 teuthology/util/test/files/test_unit_test.xml create mode 100644 teuthology/util/test/files/test_valgrind.xml create mode 100644 teuthology/util/test/test_scanner.py diff --git a/teuthology/orchestra/test/test_remote.py b/teuthology/orchestra/test/test_remote.py index 76eae68bad..a953835e7c 100644 --- a/teuthology/orchestra/test/test_remote.py +++ b/teuthology/orchestra/test/test_remote.py @@ -1,10 +1,12 @@ from mock import patch, Mock, MagicMock +from pytest import raises from io import BytesIO from teuthology.orchestra import remote from teuthology.orchestra import opsys from teuthology.orchestra.run import RemoteProcess +from teuthology.exceptions import CommandFailedError, UnitTestError class TestRemote(object): @@ -66,6 +68,23 @@ def test_run(self): assert result is proc assert result.remote is rem + @patch('teuthology.util.scanner.UnitTestScanner.scan_and_write') + def test_run_unit_test(self, m_scan_and_write): + m_transport = MagicMock() + m_transport.getpeername.return_value = ('name', 22) + self.m_ssh.get_transport.return_value = m_transport + m_run = MagicMock(name="run", side_effect=CommandFailedError('mocked error', 1, 'smithi')) + args = [ + 'something', + 'more', + ] + rem = remote.Remote(name='jdoe@xyzzy.example.com', ssh=self.m_ssh) + rem._runner = m_run + m_scan_and_write.return_value = "Error Message" + with raises(UnitTestError) as exc: + rem.run_unit_test(args=args, xml_path_regex="xml_path", output_yaml="yaml_path") + assert str(exc.value) == "Unit test failed on smithi with status 1: 'Error Message'" + def test_hostname(self): m_transport = MagicMock() m_transport.getpeername.return_value = ('name', 22) diff --git a/teuthology/orchestra/test/xml_files/test_scan_nose.xml b/teuthology/orchestra/test/xml_files/test_scan_nose.xml deleted file mode 100644 index d0fab48161..0000000000 --- a/teuthology/orchestra/test/xml_files/test_scan_nose.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - -> begin captured logging << -------------------- -botocore.hooks: DEBUG: Event choose-service-name: calling handler -PUT -/test-client.0-2txq2dyjghs0vdf-335 - -host:smithi196.front.sepia.ceph.com -x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -x-amz-date:20220929T065029Z - -host;x-amz-content-sha256;x-amz-date -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -botocore.auth: DEBUG: StringToSign: -AWS4-HMAC-SHA256 -20220929T065029Z -20220929/us-east-1/s3/aws4_request -ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb -botocore.auth: DEBUG: Signature: -8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 -botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler -botocore.endpoint: DEBUG: Sending http request: -urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 -urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 -botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} -botocore.parsers: DEBUG: Response body: -b'' -botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler -botocore.retryhandler: DEBUG: No retry needed. -GET -/test-client.0-2txq2dyjghs0vdf-335 -tagging= -host:smithi196.front.sepia.ceph.com -x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -x-amz-date:20220929T065029Z - -host;x-amz-content-sha256;x-amz-date -e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -botocore.auth: DEBUG: StringToSign: -AWS4-HMAC-SHA256 -20220929T065029Z -20220929/us-east-1/s3/aws4_request -8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 -botocore.auth: DEBUG: Signature: -a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 -botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler -botocore.endpoint: DEBUG: Sending http request: -urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 -urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 -botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} -botocore.parsers: DEBUG: Response body: -b'NoSuchTagSetErrortest-client.0-2txq2dyjghs0vdf-335tx00000ebc589e4bcad8d86-0063354035-1157-default1157-default-default' -botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler -botocore.retryhandler: DEBUG: No retry needed. -botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler > ---------------------- >> end captured logging << ---------------------]]> - \ No newline at end of file diff --git a/teuthology/util/test/files/test_unit_test.xml b/teuthology/util/test/files/test_unit_test.xml new file mode 100644 index 0000000000..bd9c73434c --- /dev/null +++ b/teuthology/util/test/files/test_unit_test.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/teuthology/util/test/files/test_valgrind.xml b/teuthology/util/test/files/test_valgrind.xml new file mode 100644 index 0000000000..41bf8375fd --- /dev/null +++ b/teuthology/util/test/files/test_valgrind.xml @@ -0,0 +1,31 @@ + + + + 0x870fc + 1 + Leak_DefinitelyLost + + 1,234 bytes in 1 blocks are definitely lost in loss record 198 of 201 + 1234 + 1 + + + + 0x4C39B6F + /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so + operator new[](unsigned long) + /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc + vg_replace_malloc.c + 640 + + + 0xF3F4B5 + /usr/bin/ceph-osd + ceph::common::leak_some_memory() + /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common + ceph_context.cc + 510 + + + + diff --git a/teuthology/util/test/test_scanner.py b/teuthology/util/test/test_scanner.py new file mode 100644 index 0000000000..1c7c89faa5 --- /dev/null +++ b/teuthology/util/test/test_scanner.py @@ -0,0 +1,191 @@ +from mock import patch, MagicMock + +from io import BytesIO +import os, io + +from teuthology.orchestra import remote +from teuthology.util.scanner import UnitTestScanner, ValgrindScanner + + +class MockFile(io.StringIO): + def close(self): + pass + + +class TestUnitTestScanner(object): + + def setup_method(self): + self.remote = remote.Remote( + name='jdoe@xyzzy.example.com', ssh=MagicMock()) + self.test_values = { + "xml_path": os.path.dirname(__file__) + "/files/test_unit_test.xml", + "error_msg": "FAILURE: Test `test_set_bucket_tagging` of `s3tests_boto3.functional.test_s3`. \ +Reason: 'NoSuchTagSetError' != 'NoSuchTagSet'.", + "summary_data": [{'failed_testsuites': {'s3tests_boto3.functional.test_s3': + [{'kind': 'failure', 'testcase': 'test_set_bucket_tagging', + 'message': "'NoSuchTagSetError' != 'NoSuchTagSet'"}]}, + 'num_of_failures': 1, + 'file_path': f'{os.path.dirname(__file__)}/files/test_unit_test.xml'}], + "yaml_data": r"""- failed_testsuites: + s3tests_boto3.functional.test_s3: + - kind: failure + message: '''NoSuchTagSetError'' != ''NoSuchTagSet''' + testcase: test_set_bucket_tagging + file_path: {file_dir}/files/test_unit_test.xml + num_of_failures: 1 +""".format(file_dir=os.path.dirname(__file__)) + } + + @patch('teuthology.util.scanner.UnitTestScanner.write_summary') + def test_scan_and_write(self, m_write_summary): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + result = UnitTestScanner(remote=self.remote).scan_and_write(xml_path, "test_summary.yaml") + assert result == self.test_values["error_msg"] + + def test_parse(self): + xml_content = b'\n\n\n' + scanner = UnitTestScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'FAILURE: Test `abc` of `xyz`. Reason: error_msg.', + {'failed_testsuites': {'xyz': + [{'kind': 'failure','message': 'error_msg','testcase': 'abc'}]}, + 'num_of_failures': 1 + } + ) + + def test_scan_file(self): + xml_path = self.test_values["xml_path"] + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = UnitTestScanner(remote=self.remote) + result = scanner.scan_file(xml_path) + assert result == self.test_values["error_msg"] + assert scanner.summary_data == self.test_values["summary_data"] + + def test_scan_all_files(self): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = UnitTestScanner(remote=self.remote) + result = scanner.scan_all_files(xml_path) + assert result == [self.test_values["error_msg"]] + + @patch('builtins.open') + def test_write_summary(self, m_open): + scanner = UnitTestScanner(self.remote) + mock_yaml_file = MockFile() + scanner.summary_data = self.test_values["summary_data"] + m_open.return_value = mock_yaml_file + scanner.write_summary("path/file.yaml") + written_content = mock_yaml_file.getvalue() + assert written_content == self.test_values["yaml_data"] + + +class TestValgrindScanner(object): + + def setup_method(self): + self.remote = remote.Remote( + name='jdoe@xyzzy.example.com', ssh=MagicMock()) + self.test_values = { + "xml_path": os.path.dirname(__file__) + "/files/test_valgrind.xml", + "error_msg": "valgrind error: Leak_DefinitelyLost\noperator new[]\ +(unsigned long)\nceph::common::leak_some_memory()", + "summary_data": [{'kind': 'Leak_DefinitelyLost', 'traceback': [{'file': + '/builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c', + 'line': '640', 'function': 'operator new[](unsigned long)'}, + {'file': '/usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc', + 'line': '510', 'function': 'ceph::common::leak_some_memory()'}], 'file_path': + f'{os.path.dirname(__file__)}/files/test_valgrind.xml'}], + "yaml_data": r"""- file_path: {file_dir}/files/test_valgrind.xml + kind: Leak_DefinitelyLost + traceback: + - file: /builddir/build/BUILD/valgrind-3.19.0/coregrind/m_replacemalloc/vg_replace_malloc.c + function: operator new[](unsigned long) + line: '640' + - file: /usr/src/debug/ceph-18.0.0-5567.g64a4fc94.el8.x86_64/src/common/ceph_context.cc + function: ceph::common::leak_some_memory() + line: '510' +""".format(file_dir=os.path.dirname(__file__)) + } + + def test_parse_with_traceback(self): + xml_content = b''' + + + Leak_DefinitelyLost + + + func() + /dir + file1.ext + 640 + + + + +''' + scanner = ValgrindScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'valgrind error: Leak_DefinitelyLost\nfunc()', + {'kind': 'Leak_DefinitelyLost', 'traceback': + [{'file': '/dir/file1.ext', 'line': '640', 'function': 'func()'}] + } + ) + + def test_parse_without_trackback(self): + xml_content = b''' + + + Leak_DefinitelyLost + + + + +''' + scanner = ValgrindScanner(self.remote) + result = scanner._parse(xml_content) + assert result == ( + 'valgrind error: Leak_DefinitelyLost\n', + {'kind': 'Leak_DefinitelyLost', 'traceback': []} + ) + + def test_scan_file(self): + xml_path = self.test_values["xml_path"] + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = ValgrindScanner(remote=self.remote) + result = scanner.scan_file(xml_path) + assert result == self.test_values["error_msg"] + assert scanner.summary_data == self.test_values["summary_data"] + + def test_scan_all_files(self): + xml_path = self.test_values["xml_path"] + self.remote.ssh.exec_command.return_value = (None, BytesIO(xml_path.encode('utf-8')), None) + m_open = MagicMock() + m_open.return_value = open(xml_path, "rb") + self.remote._sftp_open_file = m_open + scanner = ValgrindScanner(remote=self.remote) + result = scanner.scan_all_files(xml_path) + assert result == [self.test_values["error_msg"]] + + @patch('builtins.open') + def test_write_summary(self, m_open): + scanner = ValgrindScanner(self.remote) + mock_yaml_file = MockFile() + scanner.summary_data = self.test_values["summary_data"] + m_open.return_value = mock_yaml_file + scanner.write_summary("path/file.yaml") + written_content = mock_yaml_file.getvalue() + assert written_content == self.test_values["yaml_data"] \ No newline at end of file