From de17fdf3260c385012a2bdbcebe415781406050a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Tue, 21 Jan 2025 05:05:28 +0100 Subject: [PATCH] tests: add qrexec performance tests Add simple connection latency, and throughput tests. Run them with different type of services (scripts, socket, via fork-server or not). They print a test run time for comparison - the lower the better. The tests can be also started outside of the full test run by calling /usr/lib/qubes/tests/qrexec_perf.py. It requires giving names of two existing and running VMs. QubesOS/qubes-issues#5740 --- Makefile | 2 + qubes/tests/__init__.py | 1 + qubes/tests/integ/qrexec_perf.py | 117 ++++++++++++++ rpm_spec/core-dom0.spec.in | 2 + tests/qrexec_perf.py | 253 +++++++++++++++++++++++++++++++ 5 files changed, 375 insertions(+) create mode 100644 qubes/tests/integ/qrexec_perf.py create mode 100755 tests/qrexec_perf.py diff --git a/Makefile b/Makefile index 5b5d78318..852186ff4 100644 --- a/Makefile +++ b/Makefile @@ -235,6 +235,8 @@ endif mkdir -p "$(DESTDIR)$(FILESDIR)" cp -r templates "$(DESTDIR)$(FILESDIR)/templates" cp -r tests-data "$(DESTDIR)$(FILESDIR)/tests-data" + mkdir -p "$(DESTDIR)/usr/lib/qubes" + cp -r tests "$(DESTDIR)/usr/lib/qubes/" rm -f "$(DESTDIR)$(FILESDIR)/templates/README" mkdir -p "$(DESTDIR)$(DOCDIR)" diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 1fd5894df..628f45320 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -1825,6 +1825,7 @@ def load_tests(loader, tests, pattern): # pylint: disable=unused-argument "qubes.tests.integ.devices_block", "qubes.tests.integ.devices_pci", "qubes.tests.integ.qrexec", + "qubes.tests.integ.qrexec_perf", "qubes.tests.integ.dom0_update", "qubes.tests.integ.vm_update", "qubes.tests.integ.network", diff --git a/qubes/tests/integ/qrexec_perf.py b/qubes/tests/integ/qrexec_perf.py new file mode 100644 index 000000000..bc786e998 --- /dev/null +++ b/qubes/tests/integ/qrexec_perf.py @@ -0,0 +1,117 @@ +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2025 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program; if not, see . + +import asyncio +import os +import subprocess +import sys +import time + +import qubes.tests + + +class TC_00_QrexecPerfMixin: + def setUp(self: qubes.tests.SystemTestCase): + super().setUp() + self.vm1 = self.app.add_new_vm( + "AppVM", + name=self.make_vm_name("vm1"), + label="red", + ) + self.vm2 = self.app.add_new_vm( + "AppVM", + name=self.make_vm_name("vm2"), + label="red", + ) + self.loop.run_until_complete( + asyncio.gather( + self.vm1.create_on_disk(), + self.vm2.create_on_disk(), + ) + ) + self.loop.run_until_complete( + asyncio.gather( + self.vm1.start(), + self.vm2.start(), + ) + ) + + def run_test(self, name): + cmd = [ + "/usr/lib/qubes/tests/qrexec_perf.py", + f"--vm1={self.vm1.name}", + f"--vm2={self.vm2.name}", + name, + ] + p = self.loop.run_until_complete(asyncio.create_subprocess_exec(*cmd)) + self.loop.run_until_complete(p.wait()) + if p.returncode: + self.fail(f"'{' '.join(cmd)}' failed: {p.returncode}") + + def test_000_simple(self): + """Measure simple exec-based vm-vm calls latency""" + self.loop.run_until_complete(self.wait_for_session(self.vm2)) + self.run_test("exec") + + def test_010_simple_root(self): + """Measure simple exec-based vm-vm calls latency, use root to + bypass qrexec-fork-server""" + self.run_test("exec-root") + + def test_020_socket(self): + """Measure simple socket-based vm-vm calls latency""" + self.run_test("socket") + + def test_030_socket_root(self): + """Measure simple socket-based vm-vm calls latency, use root to + bypass qrexec-fork-server""" + self.run_test("socket-root") + + def test_100_simple_data_simplex(self): + """Measure simple exec-based vm-vm calls throughput""" + self.run_test("exec-data-simplex") + + def test_110_simple_data_duplex(self): + """Measure simple exec-based vm-vm calls throughput""" + self.run_test("exec-data-duplex") + + def test_120_simple_data_duplex_root(self): + """Measure simple exec-based vm-vm calls throughput""" + self.run_test("exec-data-duplex-root") + + def test_130_socket_data_duplex(self): + """Measure simple socket-based vm-vm calls throughput""" + self.run_test("socket-data-duplex") + + +def create_testcases_for_templates(): + return qubes.tests.create_testcases_for_templates( + "TC_00_QrexecPerf", + TC_00_QrexecPerfMixin, + qubes.tests.SystemTestCase, + module=sys.modules[__name__], + ) + + +def load_tests(loader, tests, pattern): + tests.addTests(loader.loadTestsFromNames(create_testcases_for_templates())) + return tests + + +qubes.tests.maybe_create_testcases_on_import(create_testcases_for_templates) diff --git a/rpm_spec/core-dom0.spec.in b/rpm_spec/core-dom0.spec.in index 9d9a21bd1..2ea3d2eb1 100644 --- a/rpm_spec/core-dom0.spec.in +++ b/rpm_spec/core-dom0.spec.in @@ -526,6 +526,7 @@ done %{python3_sitelib}/qubes/tests/integ/grub.py %{python3_sitelib}/qubes/tests/integ/salt.py %{python3_sitelib}/qubes/tests/integ/qrexec.py +%{python3_sitelib}/qubes/tests/integ/qrexec_perf.py %{python3_sitelib}/qubes/tests/integ/storage.py %{python3_sitelib}/qubes/tests/integ/vm_qrexec_gui.py @@ -547,6 +548,7 @@ done /usr/lib/qubes/cleanup-dispvms /usr/lib/qubes/fix-dir-perms.sh /usr/lib/qubes/startup-misc.sh +/usr/lib/qubes/tests/qrexec_perf.py %{_unitdir}/lvm2-pvscan@.service.d/30_qubes.conf %{_unitdir}/qubes-core.service %{_unitdir}/qubes-qmemman.service diff --git a/tests/qrexec_perf.py b/tests/qrexec_perf.py new file mode 100755 index 000000000..acca61e58 --- /dev/null +++ b/tests/qrexec_perf.py @@ -0,0 +1,253 @@ +#!/usr/bin/python3 +# +# The Qubes OS Project, https://www.qubes-os.org/ +# +# Copyright (C) 2025 Marek Marczykowski-Górecki +# +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with this program; if not, see . + +import argparse +import contextlib +import os +import time +import subprocess +import dataclasses +from typing import Union + +import qubesadmin + + +@dataclasses.dataclass +class TestConfig: + name: str + #: socket or exec + socket: bool + #: latency or throughput test + throughput: bool + #: user (via fork-server) or root + root: bool + #: for throughput test: simplex/duplex + duplex: bool + #: service file content or socat command + service: Union[bytes, list[str]] + + +latency_service_exec = b"#!/bin/sh\necho test" +latency_service_socat = [ + "socat", + "UNIX-LISTEN:/etc/qubes-rpc/test.Echo,mode=0666,fork", + "EXEC:/bin/echo test", +] +throughput_service_exec_simplex = b"#!/bin/sh\nhead -c 100000000 /dev/zero\n" +throughput_service_exec_duplex = b"#!/bin/sh\ncat\n" +throughput_service_socat_duplex = [ + "socat", + "UNIX-LISTEN:/etc/qubes-rpc/test.Echo,mode=0666,fork", + "EXEC:/bin/cat", +] + +all_tests = [ + TestConfig("exec", False, False, False, False, latency_service_exec), + TestConfig("exec-root", False, False, True, False, latency_service_exec), + TestConfig("socket", True, False, False, False, latency_service_socat), + TestConfig("socket-root", True, False, True, False, latency_service_socat), + TestConfig( + "exec-data-simplex", + False, + True, + False, + False, + throughput_service_exec_simplex, + ), + TestConfig( + "exec-data-duplex", + False, + True, + False, + True, + throughput_service_exec_simplex, + ), + TestConfig( + "exec-data-duplex-root", + False, + True, + True, + True, + throughput_service_exec_duplex, + ), + TestConfig( + "socket-data-duplex", + True, + True, + False, + True, + throughput_service_socat_duplex, + ), +] + +policy_file = "/run/qubes/policy.d/10-test-qrexec.policy" + +parser = argparse.ArgumentParser( + epilog="You can set QUBES_TEST_PERF_FILE env variable to a path where " + "machine-readable results should be saved.") +parser.add_argument("--vm1", required=True) +parser.add_argument("--vm2", required=True) +parser.add_argument( + "--iterations", + default=os.environ.get("QUBES_TEST_ITERATIONS", "500"), + type=int, +) +parser.add_argument("test", choices=[t.name for t in all_tests] + ["all"]) + + +class TestRun: + def __init__(self, vm1, vm2): + self.vm1 = vm1 + self.vm2 = vm2 + self.iterations = 500 + + def run_latency_calls(self): + start_time = time.clock_gettime(time.CLOCK_MONOTONIC) + try: + self.vm1.run( + f"set -e;" + f"for i in $(seq {self.iterations}); do " + f" out=$(qrexec-client-vm {self.vm2.name} test.Echo);" + f" test \"$out\" = 'test';" + f"done" + ) + except subprocess.CalledProcessError as e: + raise Exception( + f"test.Echo service failed ({e.returncode}):" + f" {e.stdout}," + f" {e.stderr}" + ) + end_time = time.clock_gettime(time.CLOCK_MONOTONIC) + return end_time - start_time + + def run_throughput_calls(self, duplex=False): + prefix = "" + if duplex: + prefix = "head -c 100000000 /dev/zero | " + start_time = time.clock_gettime(time.CLOCK_MONOTONIC) + try: + self.vm1.run( + f"set -e;" + f"for i in $(seq {self.iterations//2}); do " + f" out=$({prefix}qrexec-client-vm {self.vm2.name} test.Echo " + f"| wc -c);" + f' test "$out" = \'100000000\' || {{ echo "failed iteration $i:' + f" '$out'\"; exit 1; }};" + f"done" + ) + except subprocess.CalledProcessError as e: + raise Exception( + f"test.Echo service failed ({e.returncode}):" + f" {e.stdout}," + f" {e.stderr}" + ) + end_time = time.clock_gettime(time.CLOCK_MONOTONIC) + return end_time - start_time + + def report_result(self, test_name, result): + print(f"Run time ({test_name}): {result}s") + results_file = os.environ.get("QUBES_TEST_PERF_FILE") + if results_file: + try: + if self.vm1.template != self.vm2.template: + name_prefix = ( + f"{self.vm1.template!s}_" f"{self.vm2.template!s}_" + ) + else: + name_prefix = f"{self.vm1.template!s}_" + except AttributeError: + name_prefix = f"{self.vm1!s}_{self.vm2!s}_" + with open(results_file, "a") as f: + f.write(name_prefix + test_name + " " + str(result) + "\n") + + def run_test(self, test: TestConfig): + if test.root: + policy_action = "allow user=root" + else: + policy_action = "allow" + + service_proc = None + if not test.socket: + self.vm2.run_with_args( + "rm", "-f", "/etc/qubes-rpc/test.Echo", user="root" + ) + self.vm2.run_with_args( + "tee", + "/etc/qubes-rpc/test.Echo", + input=test.service, + user="root", + ) + self.vm2.run_with_args( + "chmod", "+x", "/etc/qubes-rpc/test.Echo", user="root" + ) + else: + self.vm2.run_with_args( + "tee", + "/etc/qubes/rpc-config/test.Echo", + user="root", + input=b"skip-service-descriptor=true\n", + ) + cmd = qubesadmin.utils.encode_for_vmexec(test.service) + service_proc = self.vm2.run_service( + "qubes.VMExec+" + cmd, user="root" + ) + # wait for socat startup + self.vm2.run( + "while ! test -e /etc/qubes-rpc/test.Echo; do sleep 0.1; done" + ) + + with open(policy_file, "w") as p: + p.write( + f"test.Echo + {self.vm1.name} {self.vm2.name} {policy_action}\n" + ) + try: + if test.throughput: + result = self.run_throughput_calls(test.duplex) + else: + result = self.run_latency_calls() + self.report_result(test.name, result) + finally: + os.unlink(policy_file) + if service_proc: + with contextlib.suppress(subprocess.CalledProcessError): + self.vm2.run_with_args("pkill", "socat", user="root") + service_proc.wait() + + +def main(): + args = parser.parse_args() + + if args.test == "all": + tests = all_tests + else: + tests = [t for t in all_tests if t.name == args.test] + + app = qubesadmin.Qubes() + + run = TestRun(app.domains[args.vm1], app.domains[args.vm2]) + if args.iterations: + run.iterations = args.iterations + + for test in tests: + run.run_test(test) + + +if __name__ == "__main__": + main()