diff --git a/.github/workflows/litevm.yml b/.github/workflows/litevm.yml index dbc830b3..711e09ea 100644 --- a/.github/workflows/litevm.yml +++ b/.github/workflows/litevm.yml @@ -20,29 +20,22 @@ jobs: - name: Run pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - # OL7 and OL8 use 3.6. OL9 uses 3.9. Also test on the latest. - python-minor: ["6", "9", "12"] + # Our minimum supported version of Python is 3.6, used by Oracle Linux 7 + # & 8. Ideally we would run tests on that, along with Python 3.9 for + # Oracle Linux 9, and maybe Python 3.12 for an upcoming Oracle Linux 10. + # However, practicality rules here. The binutils provided in Ubuntu + # 20.04 is not recent enough for our libctf usage, but 20.04 is the only + # remaining Ubuntu image with Python 3.6 available in Github actions. + # So we have to eliminate Python 3.6 from our test matrix. Other CI + # tests do run on Python 3.6, and there is the "vermin" pre-commit hook + # which detects code incompatible with 3.6. + python-minor: ["9", "12"] fail-fast: false - env: - # Manually set the tox environment list to the Python version. - # As a result, we should set "skip missing interpreters" to false, - # so that we fail if the test doesn't run. - TOX_OVERRIDE: "tox.envlist=py3${{ matrix.python-minor }}" - TOX_SKIP_MISSING_INTERPRETERS: "false" steps: - uses: actions/checkout@v4 - # If we rely on using the same Python version for tox as we do for the - # testing version of Python, we end up getting different versions of tox - # with different behavior. Let's setup a well-defined python version for - # tox and use that. We need to place this one before the regular python - # setup so that the regular python takes precedence. - - name: Set up Python for Tox - uses: actions/setup-python@v4 - with: - python-version: 3.12 - name: Set up Python uses: actions/setup-python@v4 with: @@ -50,7 +43,10 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install qemu-kvm zstd gzip bzip2 cpio busybox-static fio + sudo apt-get install qemu-kvm zstd gzip bzip2 cpio busybox-static fio \ + autoconf automake check gcc git liblzma-dev \ + libelf-dev libdw-dev libtool make pkgconf zlib1g-dev \ + binutils-dev sudo wget -O /usr/bin/rpm2cpio https://github.com/rpm-software-management/rpm/raw/rpm-4.19.0-release/scripts/rpm2cpio.sh echo '0403da24a797ccfa0cfd37bd4a6d6049370b9773e558da6173ae6ad25f97a428 /usr/bin/rpm2cpio' | sha256sum -c - sudo chmod 755 /usr/bin/rpm2cpio @@ -58,11 +54,15 @@ jobs: # Pinned virtualenv and tox are for the last versions which support # detecting Python 3.6 and running tests on it. run: | - python3.12 -m pip install --user --break-system-packages 'virtualenv<20.22.0' 'tox<4.5.0' - sed -i 's/sitepackages = true/sitepackages = false/' tox.ini - tox list - tox --notest - tox -e runner --notest + python -m venv venv + venv/bin/pip install -r testing/requirements-litevm.txt + venv/bin/pip install setuptools + - name: Build and install drgn with CTF support + run: | + cd .. + git clone https://github.com/brenns10/drgn -b ctf_0.0.29 + cd drgn + ../drgn-tools/venv/bin/python setup.py install - name: Run tests run: | - tox -e runner -- python -m testing.litevm.vm --delete-after-test --python-version 3${{ matrix.python-minor }} + venv/bin/python -m testing.litevm.vm --delete-after-test --with-ctf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2dfd79ba..417b077a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,11 +4,11 @@ vmtest: # time. resource_group: VM script: - - rm -rf .tox + - python -m venv venv --system-site-packages + - venv/bin/pip install -r testing/requirements-heavyvm.txt - git archive HEAD -o archive.tar.gz - - tox -e runner --notest - mkdir -p tmp/overlays tmp/info - - tox -e runner -- python -m testing.heavyvm.runner --image-dir /var/drgn-tools/images --vm-info-dir tmp/info --overlay-dir tmp/overlays --tarball archive.tar.gz + - venv/bin/python -m testing.heavyvm.runner --image-dir /var/drgn-tools/images --vm-info-dir tmp/info --overlay-dir tmp/overlays --tarball archive.tar.gz artifacts: when: always paths: @@ -18,9 +18,9 @@ vmtest: vmcore DWARF: script: - - rm -rf .tox - - tox -e runner --notest - - tox -e runner -- python -m testing.vmcore test -e py39 -j 4 --core-directory /var/drgn-tools/vmcores + - python -m venv venv --system-site-packages + - venv/bin/pip install -r testing/requirements-vmcore.txt + - venv/bin/python -m testing.vmcore.test -j 4 --core-directory /var/drgn-tools/vmcores artifacts: when: always paths: @@ -30,9 +30,9 @@ vmcore DWARF: vmcore CTF: script: - - rm -rf .tox - - tox -e runner --notest - - tox -e runner -- python -m testing.vmcore test -e py39 -j 4 --ctf --core-directory /var/drgn-tools/vmcores + - python -m venv venv --system-site-packages + - venv/bin/pip install -r testing/requirements-vmcore.txt + - venv/bin/python -m testing.vmcore.test -j 4 --ctf --core-directory /var/drgn-tools/vmcores artifacts: when: always paths: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1a77ae0..3b758887 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,5 +41,5 @@ repos: rev: 0.1.1 hooks: - id: copyright-notice - exclude: (man|doc)/.* + exclude: ((man|doc)/.*|.*requirements.*\.txt) args: [--notice=.header.txt] diff --git a/Makefile b/Makefile index 1466a8cb..6324f0ea 100644 --- a/Makefile +++ b/Makefile @@ -23,15 +23,15 @@ endif .PHONY: litevm-test litevm-test: - tox --notest - -tox -e runner -- python -m testing.litevm.vm + $(PYTHON) -m testing.litevm.vm .PHONY: vmcore-test vmcore-test: - -tox -e runner -- python -m testing.vmcore test + $(PYTHON) -m testing.vmcore test +.PHONY: test test: litevm-test vmcore-test .PHONY: docs @@ -39,7 +39,7 @@ docs: @$(PYTHON) -m tox -e docs drgn_tools/_version.py: - python setup.py -V + $(PYTHON) setup.py -V .PHONY: rsync rsync: drgn_tools/_version.py diff --git a/drgn_tools/debuginfo.py b/drgn_tools/debuginfo.py index 17780440..b35bc102 100644 --- a/drgn_tools/debuginfo.py +++ b/drgn_tools/debuginfo.py @@ -675,7 +675,9 @@ def get( return cls.NO # Prior to UEK4, it is untested and will not be tested. - if kver.uek_version < 4: + # UEK4 itself has broken CTF data (e.g. struct page) and this means that + # a large majority of helpers cannot function. + if kver.uek_version <= 4: return cls.NO # The OL7 libctf version does not support CTF generated for kernels on @@ -705,9 +707,7 @@ def get( 6: (2136, 312, 2), 7: (3, 60, 2), } - if kver.uek_version == 4: - return cls.LIMITED_PROC - elif ( + if ( kver.uek_version < 8 and kver.release_tuple < kallsyms_backport[kver.uek_version] ): diff --git a/testing/README.md b/testing/README.md index 589f2527..7eb9edae 100644 --- a/testing/README.md +++ b/testing/README.md @@ -207,13 +207,13 @@ manage your vmcores locally without needing to use OCI object storage. Assuming you have created the necessary directories and files, you can use the following to run tests: - python -m testing.vmcore test + python -m testing.vmcore.test If you've configured OCI and your `VMCORE_*` variables, then you can use the uploader to upload a specific vmcore, or download all vmcores. - python -m testing.vmcore upload --upload-core $name - python -m testing.vmcore downoad + python -m testing.vmcore.manage upload --upload-core $name + python -m testing.vmcore.manage downoad Vmcores ------- diff --git a/testing/heavyvm/runner.py b/testing/heavyvm/runner.py index 59ede131..da191dfc 100644 --- a/testing/heavyvm/runner.py +++ b/testing/heavyvm/runner.py @@ -7,9 +7,9 @@ import tempfile import time import typing as t +import xml.etree.ElementTree as ET from pathlib import Path -from junitparser import JUnitXml from paramiko.client import AutoAddPolicy from paramiko.client import SSHClient @@ -20,6 +20,7 @@ from testing.util import BASE_DIR from testing.util import ci_section_end from testing.util import ci_section_start +from testing.util import combine_junit_xml @dataclasses.dataclass @@ -90,6 +91,7 @@ class TestRunner: _vms_up: bool _ssh: t.Dict[str, SSHClient] + _xml: t.Optional[ET.ElementTree] def _section_start( self, name: str, text: str, collapsed: bool = False @@ -164,6 +166,7 @@ def __init__( self.overlay_dir = overlay_dir or self.image_dir self._vms_up = False self._ssh = {} + self._xml = None self.images = images or [] if self.vm_info_file.exists(): with self.vm_info_file.open() as f: @@ -248,20 +251,19 @@ def run_cmd(self, cmd: str) -> None: print("Result:\n" + result) self._section_end("run_cmd") - def _get_result_xml(self, info: VmInfo) -> JUnitXml: + def _get_result_xml(self, info: VmInfo) -> ET.ElementTree: ssh_client = self._get_ssh(info) sftp = ssh_client.open_sftp() - # This XML contains an encoding declaration, and if we decode it, that - # will trigger an error in the xml module. Leave it as bytes, even - # though the JUnitXml.fromstring() function insists it takes a str. - xmldata = sftp.file("/root/test/test.xml").read() - return JUnitXml.fromstring(xmldata) + return ET.parse(sftp.file("/root/test/test.xml")) - def run_test(self, cmd: str) -> int: + def run_test(self, cmd: str, ctf: bool = False) -> int: fail_list = [] - xml = JUnitXml() for vm in self.vms.values(): + if vm.uek_version == 4 and ctf: + continue slug = f"ol{vm.ol_version[0]}uek{vm.uek_version}" + if ctf: + slug += "_CTF" self._section_start( f"test_{slug}", f"Running test on {slug}", collapsed=True ) @@ -273,7 +275,7 @@ def run_test(self, cmd: str) -> int: else: print("Failed.") fail_list.append(vm.name) - xml += self._get_result_xml(vm) + self._xml = combine_junit_xml(self._xml, self._get_result_xml(vm)) self._section_end(f"test_{slug}") if fail_list: print( @@ -286,7 +288,8 @@ def run_test(self, cmd: str) -> int: output = Path("heavyvm.xml") if output.exists(): output.unlink() - xml.write(str(output)) + if self._xml is not None: + self._xml.write(str(output)) return len(fail_list) @@ -330,11 +333,14 @@ def main(): ) with r: r.copy_extract_files(args.tarball) - sys.exit( - r.run_test( - "cd /root/test && python3 -m pytest --junitxml=test.xml -o junit_logging=all tests" - ) + test_cmd = ( + "cd /root/test && " + "python3 -m pytest --junitxml test.xml -o junit_logging=all tests" ) + fail_count = 0 + fail_count += r.run_test(test_cmd) + fail_count += r.run_test(test_cmd + " --ctf", ctf=True) + sys.exit(fail_count) if __name__ == "__main__": diff --git a/testing/litevm/vm.py b/testing/litevm/vm.py index 484b6838..ffd5e998 100644 --- a/testing/litevm/vm.py +++ b/testing/litevm/vm.py @@ -295,9 +295,6 @@ def find_vmlinuz(release: str, root_dir: Path) -> Path: def run_vm(kernel: TestKernel, extract_dir: Path, commands: List[List[str]]): release = kernel.latest_release() extract_dir = extract_dir / release - if not extract_dir.is_dir(): - extract_rpms(kernel.get_rpms(), extract_dir, kernel) - script = "" if commands: for argv in commands: @@ -345,38 +342,6 @@ def run_vm(kernel: TestKernel, extract_dir: Path, commands: List[List[str]]): sys.exit(1) -def run_tests_commands( - pyver: Optional[str], ctf: bool = False -) -> List[List[str]]: - tox = Path(__file__).parent.parent.parent / ".tox" - commands = [] - for venv in tox.iterdir(): - if not venv.is_dir(): - continue - if not venv.name.startswith("py"): - continue - pytest = venv / "bin/pytest" - if not pytest.exists(): - continue - cmd = [ - str(venv / "bin/python"), - "-m", - "pytest", - "tests", - "--cov=drgn_tools", - "--cov=tests", - "-rP", - ] - if ctf: - cmd.append("--ctf") - if pyver and venv.name.endswith(pyver): - return [cmd] - commands.append(cmd) - if pyver: - raise Exception(f"Could not find python version {pyver}") - return commands - - def main(): parser = argparse.ArgumentParser(description="Lite VM Runner") parser.add_argument( @@ -396,13 +361,9 @@ def main(): help="Match against the given kernel (eg *uek6*)", ) parser.add_argument( - "--python-version", - help="Only run against one python version in tox env", - ) - parser.add_argument( - "--ctf", + "--with-ctf", action="store_true", - help="Use CTF debuginfo for tests rather than DWARF", + help="Run tests with CTF in addition to DWARF", ) parser.add_argument( "--delete-after-test", @@ -416,24 +377,45 @@ def main(): ) args = parser.parse_args() if args.command: - if args.python_version: - sys.exit("Cannot specify --python-version and custom command") - commands = [args.command] + cmd = args.command else: - commands = run_tests_commands(args.python_version, args.ctf) + cmd = [ + sys.executable, + "-m", + "pytest", + "tests", + "-rP", + ] + ctf_enabled = [False] + if args.with_ctf: + ctf_enabled.append(True) for k in TEST_KERNELS: - k.cache_dir = args.yum_cache_dir if args.kernel and not fnmatch.fnmatch(k.slug(), args.kernel): continue - section_name = f"uek{k.uek_ver}" - section_text = f"Run tests on UEK{k.uek_ver}" - with ci_section(section_name, section_text): + + # Fetch the Yum index and RPMs in the CI section that says so. + k.cache_dir = args.yum_cache_dir + with ci_section(f"uek{k.uek_ver}_fetch", f"Fetching UEK {k.uek_ver}"): release = k.latest_release() extract_dir = args.extract_dir / release - run_vm(k, args.extract_dir, commands) - if args.delete_after_test: - shutil.rmtree(extract_dir) - k.delete_cache() + if not extract_dir.is_dir(): + extract_rpms(k.get_rpms(), extract_dir, k) + + for ctf in ctf_enabled: + if ctf: + section_name = f"uek{k.uek_ver}_CTF" + section_text = f"Run tests on UEK{k.uek_ver} with CTF" + # add the CTF argument here + run_cmd = cmd + ["--ctf"] + else: + section_name = f"uek{k.uek_ver}" + section_text = f"Run tests on UEK{k.uek_ver}" + run_cmd = cmd + with ci_section(section_name, section_text): + run_vm(k, args.extract_dir, [run_cmd]) + if args.delete_after_test: + shutil.rmtree(extract_dir) + k.delete_cache() if __name__ == "__main__": diff --git a/testing/parlib.py b/testing/parlib.py new file mode 100644 index 00000000..9a4ccc09 --- /dev/null +++ b/testing/parlib.py @@ -0,0 +1,455 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +""" +Pre-authenticated request library: parlib + +A simple library and tool for using OCI Object Storage Pre-authenticated +Requests (PARs). + +PARs allow you to share a URL which allows another user to access (and possibly +modify) resources in an Object Storage Bucket without the need for OCI +credentials. The permissions are relatively flexible, and the PAR can't provide +any access outside the scope of those permissions. + +While the main use for PARs is to allow a user to upload or download a single +object, they also can be used to provide RO or RW access to an entire bucket or +a prefix. The PARs expire and may be revoked at any time. This makes them ideal +for shared environments too. + +To provide this functionality, a full ListObjects API as well as GET, HEAD, and +PUT object requests are available for the PARs. Unfortunately, you can't use the +OCI SDK to perform these ListObjects requests. The recommended method is to +manually construct requests and execute them with a tool like cURL. + +This library provides a Python API to provide ListObjects as well as object +fetch and upload. It it Python 3.6 compatible, without any third-party +dependencies. +""" +import argparse +import json +import logging +import os +import posixpath +import shutil +import sys +import warnings +from http.client import HTTPResponse +from http.client import HTTPSConnection +from typing import Any +from typing import BinaryIO +from typing import Dict +from typing import Iterator +from typing import List +from typing import NamedTuple +from typing import Optional +from typing import Union +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.parse import urlparse +from urllib.request import Request +from urllib.request import urlopen + + +class ListResult(NamedTuple): + nextStartWith: Optional[str] + objects: List[Dict[str, Any]] + prefixes: List[str] + + +MAX_LIST = 1000 +MULTIPART_CHUNK = 100 * 1024 * 1024 +log = logging.getLogger(__name__) + + +class ParClient: + url: str + host: str + path: str + conn: HTTPSConnection + _last: Optional[HTTPResponse] + + def __init__(self, url: str): + parsed = urlparse(url) + if parsed.scheme != "https": + raise ValueError("PAR URLs must be https") + self.url = url + self.host = parsed.netloc + self.path = parsed.path + if "https_proxy" in os.environ: + pstr = os.environ["https_proxy"] + if "://" not in pstr: + pstr = "http://" + pstr + proxy = urlparse(pstr) + log.debug("using proxy: %s", pstr) + self.conn = HTTPSConnection(proxy.netloc) + self.conn.set_tunnel(self.host) + else: + self.conn = HTTPSConnection(self.host) + self._last = None + + def request( + self, + path: str, + query: Optional[Dict[str, str]] = None, + method: str = "GET", + headers: Optional[Dict[str, str]] = None, + data: Union[None, bytes, BinaryIO] = None, + ) -> HTTPResponse: + # We use a single HTTPSConnection for the lifetime of the client. This + # is a bit lower-level than what people normally would use (urllib). + # However, the reason makes sense: we're only making requests serially + # in one thread, to one server. In the case of multipart uploads, it's + # nice to avoid the extra overhead of establishing a new connection. + # + # For exceptionally long-lived clients, it's possible that the + # connection gets closed unexpectedly. We may may need to implement a + # way to reopen the connection. We'll wait and see whether it's + # necessary before implementing that. + # + # Due to the use of the HTTPSConnection, we need to ensure that the + # previous request has been fully completed before we issue a new one. + # If the request had a body which was not read, it's a potential error. + # However, don't raise an exception for it, because this can happen + # during some error handling paths. + if self._last: + unconsumed = len(self._last.read()) + if unconsumed: + warnings.warn( + f"Left {unconsumed} bytes unconsumed from previous request", + RuntimeWarning, + ) + if query: + path += "?" + urlencode(query) + headers = headers or {} + headers["Host"] = self.host + headers["Connection"] = "keep-alive" + self.conn.request(method, path, body=data, headers=headers) + resp = self.conn.getresponse() + self._last = resp + logging.debug("%d: %s %s", resp.status, method, path) + if not (200 <= resp.status < 300): + raise HTTPError( + f"https://{self.host}/{path}", + resp.status, + resp.reason, + hdrs=resp.headers, + fp=resp.fp, + ) + return resp + + def _request( + self, + path: str, + query: Optional[Dict[str, str]] = None, + method: str = "GET", + headers: Optional[Dict[str, str]] = None, + data: Union[None, bytes, BinaryIO] = None, + ) -> HTTPResponse: + # This is an alternative implementation of request() that uses urllib. + # It's much simpler, but of course it has extra overhead for the + # multipart uploads. It is effectively dead code but retained here for + # reference in case we need to revert to it later. + if query: + path += "?" + urlencode(query) + req = Request( + f"https://{self.host}{path}", + method=method, + headers=(headers or {}), + data=data, + ) + resp = urlopen(req) + logging.debug("%d: %s %s", resp.status, method, path) + return resp + + def __enter__(self) -> "ParClient": + return self + + def __exit__(self, *args, **kwargs): + self.conn.close() + + def list_objects_raw( + self, + prefix: Optional[str] = None, + fields: Union[None, str, List[str]] = None, + delimiter: Optional[str] = None, + limit: Optional[int] = None, + start: Optional[str] = None, + end: Optional[str] = None, + startAfter: Optional[str] = None, + ) -> ListResult: + """ + List objects in the bucket that are accessible to this PAR. + + This exposes the raw ListObjects API for the PAR, which is pretty much + the same as the standard ListObjects API. Pagination is not directly + handled here, instead it must be done manually or via + ``list_objects_paginated()``. Errors are simply raised as HTTPError from + urllib. The official OCI documentation for ListObjects should be + considered canonical, along with any adujstments described for PARs: + + https://docs.oracle.com/en-us/iaas/api/#/en/objectstorage/20160918/Object/ListObjects + https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingpreauthenticatedrequests_topic-Working_with_PreAuthenticated_Requests.htm + + :param prefix: Object prefix to filter by. Note that the PAR may have + limitations that already filters by a prefix. + :param fields: What fields should be returned for each object. Choices + are documented in the OCI docs but include name, size, etag, md5, + timeCreated, timeModified, storageTier, archivalState. + :param delimiter: When supplied, only return objects without the + delimiter in their name. Prefixes of object names up to the delimiter + are also returned. This mimics a filesystem abstraction. Only "/" is a + supported delimiter. + :param limit: Return this many objects at a maximum. Note that OCI + places a hard limit of 1000 on the response size. + :param start: Return objects lexicographically greater or equal to key. + :param end: Return objects lexicographically less or qual to key. + :param startAfter: Return objects lexicographically greater than key. + :returns: A list result, which contains a list of objects, as well as + possibly a list of prefixes (when ``delimiter`` is used), and a + ``nextStartWith`` key in case another page of results is available. + """ + query = {} + if isinstance(fields, list): + query["fields"] = ",".join(fields) + elif isinstance(fields, str): + query["fields"] = fields + + if prefix is not None: + query["prefix"] = prefix + if delimiter is not None: + query["delimiter"] = delimiter + if limit is not None: + query["limit"] = str(limit) + if start is not None: + query["limit"] = start + if end is not None: + query["end"] = end + if startAfter is not None: + query["startAfter"] = startAfter + + # There's not much point in catching and handling HTTP errors here. The + # client will probably know better what to do with them than we will. + data = json.load(self.request(self.path, query=query)) + + objects = data["objects"] + nextStartWith = data.get("nextStartWith") + prefixes = data.get("prefixes", []) + return ListResult(nextStartWith, objects, prefixes) + + def list_objects_paginated( + self, + prefix: Optional[str] = None, + fields: Union[None, str, List[str]] = None, + delimiter: Optional[str] = None, + limit: Optional[int] = MAX_LIST, + start: Optional[str] = None, + end: Optional[str] = None, + startAfter: Optional[str] = None, + ) -> Iterator[ListResult]: + """ + An automatically paginated version of ``list_objects_raw`` + + This API returns an iterator of each result from ``list_objects_raw()``, + handling the pagination for you. The parameters are the same as the + above. The ``limit`` parameter will be used as a chunk size for + pagination. + """ + while True: + res = self.list_objects_raw( + prefix=prefix, + fields=fields, + delimiter=delimiter, + limit=limit, + start=start, + end=end, + startAfter=startAfter, + ) + yield res + if res.nextStartWith is not None: + start = res.nextStartWith + else: + break + + def list_objects_simple( + self, + prefix: Optional[str] = None, + fields: Union[None, str, List[str]] = None, + limit: Optional[int] = MAX_LIST, + start: Optional[str] = None, + end: Optional[str] = None, + startAfter: Optional[str] = None, + ) -> Iterator[Dict[str, Any]]: + """ + Return a transparently paginated iterator of objects + + This is the simplest version of the ListObjects API. It does not support + the ``delimiter`` argument. It simply yields back objects, handling + pagination using the ``limit`` provided in the parameters. See the + documentation of ``list_objects_raw()`` for more details. + """ + iterator = self.list_objects_paginated( + prefix=prefix, + fields=fields, + limit=limit, + start=start, + end=end, + startAfter=startAfter, + ) + for result in iterator: + yield from result.objects + + def get_object(self, key: str) -> BinaryIO: + """ + Read an object, returning a file-like object. + """ + path = posixpath.join(self.path, key) + return self.request(path) + + def put_object_raw(self, key: str, data: bytes) -> None: + """ + Upload an object directly using a single PUT request. + + Uploads can be complex for large objects. This API does the simplest + option: it directly uploads an object using the PUT method, with all the + data at once. + """ + path = posixpath.join(self.path, key) + self.request(path, method="PUT", data=data) + + def put_object_multipart( + self, + key: str, + data: BinaryIO, + chunk_size: int = MULTIPART_CHUNK, + first_block: Optional[bytes] = None, + ) -> None: + """ + Upload an object using a multipart upload. + """ + path = posixpath.join(self.path, key) + headers = {"opc-multipart": "true"} + with self.request(path, method="PUT", headers=headers) as f: + multipart = json.load(f) + + try: + part = 1 + while True: + block = first_block or data.read(chunk_size) + first_block = None + if not block: + break + path = posixpath.join(multipart["accessUri"], str(part)) + self.request(path, method="PUT", data=block) + part += 1 + + self.request(multipart["accessUri"], method="POST") + except BaseException: + # If we're interrupted for any reason, including keyboard interrupt + # or exceptions, we need to delete the multipart upload. + self.request(multipart["accessUri"], method="DELETE") + raise + + def put_object( + self, + key: str, + data: BinaryIO, + chunk_size: int = MULTIPART_CHUNK, + ) -> None: + """ + Upload an object, selecting multipart when it is large or unknown size + """ + first_block = data.read(chunk_size) + if len(first_block) < chunk_size: + self.put_object_raw(key, data.read()) + else: + self.put_object_multipart( + key, data, chunk_size, first_block=first_block + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Pre-authenticated request tools" + ) + parser.add_argument( + "--url", + "-u", + type=str, + default=os.environ.get("OCI_PAR_URL"), + help="Pre-authenticated request URL. This can be provided " + "via the environment variable OCI_PAR_URL as well.", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="enable debug logging", + ) + parser.add_argument( + "--output", + "-o", + type=str, + default=None, + help="output file for get operation", + ) + parser.add_argument( + "--chunk-size", + "-c", + type=int, + default=MULTIPART_CHUNK, + help="Chunk size (in bytes) for multipart upload", + ) + parser.add_argument( + "operation", + choices=["list", "ls", "get", "put"], + help="Choose an operation", + ) + parser.add_argument( + "arg", + type=str, + default=None, + nargs="?", + help="Argument to operation", + ) + parser.add_argument( + "arg2", + type=str, + default=None, + nargs="?", + help="Argument 2 to operation", + ) + + args = parser.parse_args() + loglevel = logging.DEBUG if args.verbose else logging.WARNING + logging.basicConfig(level=loglevel) + client = ParClient(args.url) + if args.operation == "list": + for object in client.list_objects_simple(args.arg, fields="size"): + print(f"{object['size']:10d} {object['name']}") + elif args.operation == "ls": + for resp in client.list_objects_paginated( + args.arg, fields="size", delimiter="/" + ): + for pfx in resp.prefixes: + print(f" DIR {pfx}") + for object in resp.objects: + print(f"{object['size']:10d} {object['name']}") + elif args.operation == "get": + if not args.arg: + sys.exit("usage: get OBJECT") + if args.output: + out = open(args.output, "wb") + else: + out = os.fdopen(sys.stdout.fileno(), "wb", closefd=False) + shutil.copyfileobj(client.get_object(args.arg), out) + elif args.operation == "put": + if not args.arg and not args.arg2: + sys.exit("usage: put KEY FILENAME") + with open(args.arg2, "rb") as f: + client.put_object(args.arg, f, args.chunk_size) + else: + sys.exit(f"unknown operation: {args.operation}") + + +if __name__ == "__main__": + main() diff --git a/testing/requirements-heavyvm.txt b/testing/requirements-heavyvm.txt new file mode 100644 index 00000000..31dacc36 --- /dev/null +++ b/testing/requirements-heavyvm.txt @@ -0,0 +1,4 @@ +# Requirements for running the heavyvm tests. +# Used in Gitlab CI. +paramiko +pytest diff --git a/testing/requirements-litevm.txt b/testing/requirements-litevm.txt new file mode 100644 index 00000000..beb96ee9 --- /dev/null +++ b/testing/requirements-litevm.txt @@ -0,0 +1,3 @@ +# Requirements for running the litevm tests. +# Used in Github CI. +pytest diff --git a/testing/requirements-vmcore.txt b/testing/requirements-vmcore.txt new file mode 100644 index 00000000..72ff412c --- /dev/null +++ b/testing/requirements-vmcore.txt @@ -0,0 +1,3 @@ +# Requirements for running the vmcore tests. +# Used in Gitlab CI. +pytest diff --git a/testing/requirements.txt b/testing/requirements.txt deleted file mode 100644 index 53d9efb2..00000000 --- a/testing/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -#dataclasses # backport for <3.8 -oci>=2.23.2,<2.118.1 # upload manager thread safety, temporary pin due to python-pkcs11 build failure -oci-cli -paramiko -rich -types-paramiko -junitparser==3.1.0 diff --git a/testing/rpm.py b/testing/rpm.py index 023a4582..b38014a3 100644 --- a/testing/rpm.py +++ b/testing/rpm.py @@ -23,9 +23,7 @@ CORE_DIR = Path.cwd() / "testdata/vmcores" -def vmcore_test( - vmcore: str, ctf: bool = False, coverage: bool = False, host_ol: int = 9 -) -> str: +def vmcore_test(vmcore: str, ctf: bool = False, host_ol: int = 9) -> str: uname = CORE_DIR / vmcore / "UTS_RELEASE" with uname.open() as f: release = f.read().strip() @@ -50,26 +48,19 @@ def vmcore_test( f"--vmcore-dir={str(CORE_DIR)}", ], ctf=ctf, - coverage=coverage, ) -def do_test( - ident: str, args: List[str], ctf: bool = False, coverage: bool = False -) -> str: +def do_test(ident: str, args: List[str], ctf: bool = False) -> str: kind = "CTF" if ctf else "DWARF" print("=" * 30 + f" TESTING {ident} W/ {kind} " + "=" * 30) cmd = [ - "python3", + sys.executable, "-m", "pytest", + "tests/", ] - if coverage: - cmd += [ - "--cov=drgn_tools", - "--cov-append", - ] if ctf: cmd.append("--ctf") @@ -81,9 +72,7 @@ def do_test( return "pass" -def live_test( - ctf: bool = False, coverage: bool = False, host_ol: int = 9 -) -> str: +def live_test(ctf: bool = False, host_ol: int = 9) -> str: release = os.uname().release kind = "CTF" if ctf else "DWARF" kver = KernelVersion.parse(release) @@ -102,7 +91,7 @@ def live_test( path = f"/usr/lib/debug/lib/modules/{release}/vmlinux" if not os.path.exists(path): return f"skip ({kind} not available)" - return do_test("LIVE", [], ctf=ctf, coverage=coverage) + return do_test("LIVE", [], ctf=ctf) def osrelease() -> Dict[str, str]: @@ -139,16 +128,6 @@ def main() -> None: action="store_false", help="do not run live kernel test", ) - parser.add_argument( - "--coverage", - action="store_true", - help="run code coverage (requires pytest-cov)", - ) - parser.add_argument( - "--xml", - action="store_true", - help="collect output in XML (requires junitparser)", - ) parser.add_argument( "--core-dir", type=Path, @@ -188,35 +167,28 @@ def should_run_vmcore(name: str) -> bool: print(cores) - if args.coverage: - cov = Path.cwd() / ".coverage" - if cov.is_file(): - cov.unlink() - fail = False results = defaultdict(list) for core in cores: if args.dwarf: - res = vmcore_test(core, coverage=args.coverage, host_ol=ol_ver) + res = vmcore_test(core, host_ol=ol_ver) results[res].append(f"{core} (DWARF)") if "fail" in res or "error" in res: fail = True if args.ctf: - res = vmcore_test( - core, ctf=True, coverage=args.coverage, host_ol=ol_ver - ) + res = vmcore_test(core, ctf=True, host_ol=ol_ver) results[res].append(f"{core} (CTF)") if "fail" in res or "error" in res: fail = True if args.live: - res = live_test(coverage=args.coverage, host_ol=ol_ver) + res = live_test(host_ol=ol_ver) results[res].append("live (DWARF)") if "fail" in res or "error" in res: fail = True - res = live_test(ctf=True, coverage=args.coverage, host_ol=ol_ver) + res = live_test(ctf=True, host_ol=ol_ver) results[res].append("live (CTF)") if "fail" in res or "error" in res: fail = True diff --git a/testing/util.py b/testing/util.py index 6e8e2266..27ad3e59 100644 --- a/testing/util.py +++ b/testing/util.py @@ -2,9 +2,11 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import os import time +import xml.etree.ElementTree as ET from contextlib import contextmanager from pathlib import Path from typing import Generator +from typing import Optional BASE_DIR = (Path(__file__).parent.parent / "testdata").absolute() """ @@ -70,3 +72,25 @@ def ci_section( yield finally: ci_section_end(name) + + +def combine_junit_xml( + main: Optional[ET.ElementTree], + new: ET.ElementTree, +) -> ET.ElementTree: + """ + Combine the JUnit XML files created by pytest. While we could use the + "junitxml" PyPI package for this, all we really need to know about JUnit XML + is that there is a root element named "testsuites", child elements of tag + type "testsuite", and then children of type "testcase". + + So, to combine two files, just add its test cases into the root element. + """ + if main is None: + return new + + assert main.getroot().tag == "testsuites" + assert new.getroot().tag == "testsuites" + + main.getroot().extend(new.getroot()) + return main diff --git a/testing/vmcore.py b/testing/vmcore.py deleted file mode 100644 index b432e9a2..00000000 --- a/testing/vmcore.py +++ /dev/null @@ -1,446 +0,0 @@ -# Copyright (c) 2023, Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ -""" -Manager for test vmcores - downloaded from OCI block storage -""" -import argparse -import fnmatch -import os -import signal -import subprocess -import sys -import time -from concurrent.futures import as_completed -from concurrent.futures import ThreadPoolExecutor -from contextlib import ExitStack -from pathlib import Path -from tempfile import NamedTemporaryFile -from threading import Event -from typing import Any -from typing import List -from typing import Optional -from typing import Tuple - -import oci.config -from junitparser import JUnitXml -from oci.exceptions import ConfigFileNotFound -from oci.object_storage import ObjectStorageClient -from oci.object_storage import UploadManager -from oci.pagination import list_call_get_all_results_generator -from rich.progress import BarColumn -from rich.progress import DownloadColumn -from rich.progress import Progress -from rich.progress import TaskID -from rich.progress import TextColumn -from rich.progress import TimeRemainingColumn -from rich.progress import TransferSpeedColumn - -from drgn_tools.debuginfo import CtfCompatibility -from drgn_tools.debuginfo import KernelVersion - -CORE_DIR = Path.cwd() / "testdata/vmcores" - -CHUNK_SIZE = 16 * 4096 -UPLOAD_PART_SIZE = 16 * 1024 * 1024 - -SIGTERM_EVENT = Event() -signal.signal(signal.SIGTERM, lambda: SIGTERM_EVENT.set()) # type: ignore - - -def get_oci_bucket_info() -> Tuple[str, str, str]: - namespace = os.environ.get("VMCORE_NAMESPACE") - bucket = os.environ.get("VMCORE_BUCKET") - prefix = os.environ.get("VMCORE_PREFIX") - if not (namespace and bucket and prefix): - raise Exception( - "Please set VMCORE_NAMESPACE, VMCORE_BUCKET, and VMCORE_PREFIX to " - "point to the OCI object storage location for the vmcore repo." - ) - return namespace, bucket, prefix - - -def download_file( - client: ObjectStorageClient, - progress: Progress, - name: str, - key: str, - path: Path, - size: int, -): - progress.print(f"Downloading {name}") - task_id = progress.add_task( - "download", - filename=name, - total=size, - start=True, - ) - namespace, bucket, _ = get_oci_bucket_info() - response = client.get_object(namespace, bucket, key) - relpath = path.relative_to(CORE_DIR) - with path.open("wb") as f: - for content_bytes in response.data.iter_content(chunk_size=CHUNK_SIZE): - f.write(content_bytes) - progress.update(task_id, advance=len(content_bytes)) - if SIGTERM_EVENT.is_set(): - progress.print(f"[red]Download interrupted[/red]: {relpath}") - return - progress.print(f"Download completed: {relpath}") - progress.remove_task(task_id) - - -def all_objects(client: ObjectStorageClient) -> List[Any]: - objects = [] - namespace, bucket, prefix = get_oci_bucket_info() - gen = list_call_get_all_results_generator( - client.list_objects, - "response", - namespace, - bucket, - prefix=prefix, - fields="size", - ) - for response in gen: - objects.extend(response.data.objects) - return objects - - -def download_all(client: ObjectStorageClient): - _, _, prefix = get_oci_bucket_info() - progress = Progress( - TextColumn("[bold blue]{task.fields[filename]}", justify="right"), - BarColumn(bar_width=None), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - DownloadColumn(), - "•", - TransferSpeedColumn(), - "•", - TimeRemainingColumn(), - ) - objects = all_objects(client) - CORE_DIR.mkdir(exist_ok=True) - with progress, ThreadPoolExecutor(max_workers=8) as pool: - futures = [] - for obj in objects: - assert obj.name.startswith(prefix) - name = obj.name[len(prefix) :] - path = CORE_DIR / name - path.parent.mkdir(parents=True, exist_ok=True) - if path.is_file() and path.stat().st_size == obj.size: - progress.print(f"Already exists: {name}") - else: - futures.append( - pool.submit( - download_file, - client, - progress, - name, - obj.name, - path, - obj.size, - ) - ) - for future in as_completed(futures): - try: - future.result() - except Exception as e: - print(e) - - -def delete_orphans(client: ObjectStorageClient): - print("Searching for orphaned files to remove...") - objs = all_objects(client) - keys = set() - _, _, prefix = get_oci_bucket_info() - for obj in objs: - assert obj.name.startswith(prefix) - name = obj.name[len(prefix) :] - keys.add(name) - - # Iterate using list() because modifying the directory while iterating it - # can lead to errors - for fn in list(CORE_DIR.glob("**/*")): - if not fn.is_file(): - continue - key = str(fn.relative_to(CORE_DIR)) - if key in keys: - continue - print(f"Remove orphaned file: {key}") - fn.unlink() - parent = fn.parent - while not list(parent.iterdir()): - print(f"Remove empty parent: {parent}") - parent.rmdir() - parent = parent.parent - - -def upload_file( - client: ObjectStorageClient, - progress: Progress, - task_id: TaskID, - key: str, - path: Path, -) -> None: - def cb(nbytes: int) -> None: - progress.update(task_id, advance=nbytes) - - namespace, bucket, _ = get_oci_bucket_info() - progress.start_task(task_id) - manager = UploadManager(client) - manager.upload_file( - namespace, - bucket, - key, - str(path), - progress_callback=cb, - ) - - -def upload_all(client: ObjectStorageClient, core: str) -> None: - _, _, prefix = get_oci_bucket_info() - progress = Progress( - TextColumn("[bold blue]{task.fields[filename]}", justify="right"), - BarColumn(bar_width=None), - "[progress.percentage]{task.percentage:>3.1f}%", - "•", - DownloadColumn(), - "•", - TransferSpeedColumn(), - "•", - TimeRemainingColumn(), - ) - core_path = CORE_DIR / core - vmlinux_path = core_path / "vmlinux" - vmcore_path = core_path / "vmcore" - if not vmlinux_path.exists() or not vmcore_path.exists(): - sys.exit("error: missing vmcore or vmlinux file") - uname = core_path / "UTS_RELEASE" - if not uname.exists(): - sys.exit("error: missing UTS_RELEASE file") - uploads = [vmlinux_path, vmcore_path, uname] - uploads += list(core_path.glob("*.ko.debug")) - uploads += list(core_path.glob("vmlinux.ctfa*")) - object_to_size = {obj.name: obj.size for obj in all_objects(client)} - with progress, ThreadPoolExecutor(max_workers=4) as pool: - futures = [] - for path in uploads: - key = prefix + str(path.relative_to(CORE_DIR)) - existing_size = object_to_size.get(key) - size = path.stat().st_size - if existing_size is not None and existing_size == size: - progress.print(f"Already uploaded: {key}") - continue - task_id = progress.add_task( - "upload", - filename=key, - total=path.stat().st_size, - start=False, - ) - fut = pool.submit( - upload_file, - client, - progress, - task_id, - key, - path, - ) - futures.append(fut) - for future in as_completed(futures): - future.result() - - -def _test_job( - core_name: str, cmd: List[str], xml: str -) -> Tuple[str, bool, JUnitXml]: - # Runs the test silently, but prints the stdout/stderr on failure. - with NamedTemporaryFile("w+t") as f: - print(f"Begin testing {core_name}") - start = time.time() - res = subprocess.run(cmd, stdout=f, stderr=f) - if res.returncode != 0: - print(f"=== FAILURE: {core_name} ===") - f.seek(0) - sys.stdout.write(f.read()) - runtime = time.time() - start - print(f"Completed testing {core_name} in {runtime:.1f}") - run_data = JUnitXml.fromfile(xml) - return (core_name, res.returncode == 0, run_data) - - -def _skip_ctf(ctf: bool, uname: str) -> bool: - if ctf: - host_ol = 9 # OL8 or 9 work here, tests aren't supported for OL7 - kver = KernelVersion.parse(uname) - compat = CtfCompatibility.get(kver, host_ol) - # Skip test when CTF is fully unsupported, or when it would require a - # /proc/kallsyms. - return compat in ( - CtfCompatibility.NO, - CtfCompatibility.LIMITED_PROC, - ) - return False # don't skip when non-CTF - - -def test( - vmcore_list: List[str], - env: Optional[str] = None, - ctf: bool = False, - parallel: int = 1, -) -> None: - def should_run_vmcore(name: str) -> bool: - if not vmcore_list: - return True - for pat in vmcore_list: - if fnmatch.fnmatch(name, pat): - return True - return False - - failed = [] - passed = [] - skipped = [] - xml = JUnitXml() - - tox_cmd = ["tox"] - if env: - tox_cmd += ["-e", env] - # Initialize the virtualenv once, and then tell tox not to do it again. - subprocess.run(tox_cmd + ["--notest"]) - tox_cmd.append("--skip-pkg-install") - - with ExitStack() as es: - pool = es.enter_context(ThreadPoolExecutor(max_workers=parallel)) - futures = [] - for path in CORE_DIR.iterdir(): - core_name = path.name - if not should_run_vmcore(core_name): - continue - uname = (path / "UTS_RELEASE").read_text().strip() - if _skip_ctf(ctf, uname): - skipped.append(core_name) - continue - xml_run = es.enter_context( - NamedTemporaryFile("w", suffix=".xml", delete=False) - ) - xml_run.close() # not deleted until context is ended - cmd = tox_cmd + [ - "--", - f"--vmcore={core_name}", - f"--vmcore-dir={str(CORE_DIR)}", - f"--junitxml={xml_run.name}", - "-o", - "junit_logging=all", - ] - if ctf: - if not (path / "vmlinux.ctfa").is_file(): - skipped.append(core_name) - continue - cmd.append("--ctf") - futures.append( - pool.submit(_test_job, core_name, cmd, xml_run.name) - ) - - for future in futures: - core_name, test_passed, run_data = future.result() - xml += run_data - if test_passed: - passed.append(core_name) - else: - failed.append(core_name) - - xml.write("vmcore.xml") - print("Complete test logs: vmcore.xml") - print("Vmcore Test Summary -- Passed:") - print("\n".join(f"- {n}" for n in passed)) - if skipped: - print("Vmcore Test Summary -- Skipped (missing CTF):") - print("\n".join(f"- {n}" for n in skipped)) - if failed: - print("Vmcore Test Summary -- FAILED:") - print("\n".join(f"- {n}" for n in failed)) - sys.exit(1) - - -def get_client() -> ObjectStorageClient: - try: - config = oci.config.from_file() - return ObjectStorageClient(config) - except ConfigFileNotFound: - sys.exit( - "error: You need to configure OCI!\n" - 'Try running ".tox/runner/bin/oci setup bootstrap"' - ) - - -def main(): - global CORE_DIR - parser = argparse.ArgumentParser( - description="manages drgn-tools vmcores", - ) - parser.add_argument( - "action", - choices=["download", "upload", "test"], - help="choose which operation", - ) - parser.add_argument( - "--upload-core", - type=str, - help="choose name of the vmcore to upload", - ) - parser.add_argument( - "--core-directory", type=Path, help="where to store vmcores" - ) - parser.add_argument( - "--delete-orphan", - action="store_true", - help="delete any files which are not listed on block storage", - ) - parser.add_argument( - "--vmcore", - action="append", - default=[], - help="only run tests on the given vmcore(s). you can use this " - "multiple times to specify multiple vmcore names. You can also " - "use fnmmatch patterns to specify several cores at once.", - ) - parser.add_argument( - "--tox-env", - "-e", - default=None, - help="run tests within this tox environment", - ) - parser.add_argument( - "--ctf", - action="store_true", - help="Use CTF debuginfo for tests rather than DWARF (skips vmcores " - "without a vmlinux.ctfa file)", - ) - parser.add_argument( - "--parallel", - "-j", - type=int, - default=1, - help="Run the tests in parallel with the given number of threads", - ) - args = parser.parse_args() - if args.core_directory: - CORE_DIR = args.core_directory.absolute() - if args.action == "download": - client = get_client() - download_all(client) - if args.delete_orphan: - delete_orphans(client) - elif args.action == "upload": - if not args.upload_core: - sys.exit("error: --upload-core is required for upload operation") - upload_all(get_client(), args.upload_core) - elif args.action == "test": - test( - args.vmcore, - env=args.tox_env, - ctf=args.ctf, - parallel=args.parallel, - ) - - -if __name__ == "__main__": - main() diff --git a/testing/vmcore/__init__.py b/testing/vmcore/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/vmcore/manage.py b/testing/vmcore/manage.py new file mode 100644 index 00000000..9aecfc98 --- /dev/null +++ b/testing/vmcore/manage.py @@ -0,0 +1,146 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +""" +Download, upload, and synchronize a collection of vmcores +""" +import argparse +import os +import shutil +import sys +from pathlib import Path + +from testing.parlib import ParClient +from testing.util import BASE_DIR + +# May be overridden by the CLI +CORE_DIR = BASE_DIR / "vmcores" +CORE_PFX = "drgn-tools-vmcores/" + + +def download_all(client: ParClient): + objects = client.list_objects_simple(prefix=CORE_PFX, fields="size") + CORE_DIR.mkdir(exist_ok=True) + for obj in objects: + name = obj["name"][len(CORE_PFX) :] + path = CORE_DIR / name + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file() and path.stat().st_size == obj["size"]: + print(f"Already exists: {name}") + else: + print(f"Download: {name}") + with path.open("wb") as f: + shutil.copyfileobj(client.get_object(obj["name"]), f) + + +def delete_orphans(client: ParClient): + print("Searching for orphaned files to remove...") + objs = client.list_objects_simple(prefix=CORE_PFX, fields="size") + keys = set() + for obj in objs: + assert obj["name"].startswith(CORE_PFX) + name = obj["name"][len(CORE_PFX) :] + keys.add(name) + + # Iterate using list() because modifying the directory while iterating it + # can lead to errors + for fn in list(CORE_DIR.glob("**/*")): + if not fn.is_file(): + continue + key = str(fn.relative_to(CORE_DIR)) + if key in keys: + continue + print(f"Remove orphaned file: {key}") + fn.unlink() + parent = fn.parent + while not list(parent.iterdir()): + print(f"Remove empty parent: {parent}") + parent.rmdir() + parent = parent.parent + + +def upload_all(client: ParClient, core: str) -> None: + core_path = CORE_DIR / core + vmlinux_path = core_path / "vmlinux" + vmcore_path = core_path / "vmcore" + if not vmlinux_path.exists() or not vmcore_path.exists(): + sys.exit("error: missing vmcore or vmlinux file") + uname = core_path / "UTS_RELEASE" + if not uname.exists(): + sys.exit("error: missing UTS_RELEASE file") + uploads = [vmlinux_path, vmcore_path, uname] + uploads += list(core_path.glob("*.ko.debug")) + uploads += list(core_path.glob("vmlinux.ctfa*")) + object_to_size = { + obj["name"]: obj["size"] + for obj in client.list_objects_simple(fields="size") + } + + for path in uploads: + key = CORE_PFX + str(path.relative_to(CORE_DIR)) + existing_size = object_to_size.get(key) + size = path.stat().st_size + if existing_size is not None and existing_size == size: + print(f"Already uploaded: {key}") + continue + with path.open("rb") as f: + print(f"Upload: {key}") + client.put_object(key, f) + + +def main(): + global CORE_DIR, CORE_PFX + parser = argparse.ArgumentParser( + description="manages drgn-tools vmcores", + ) + parser.add_argument( + "action", + choices=["download", "upload"], + help="choose which operation", + ) + parser.add_argument( + "--upload-core", + type=str, + help="choose name of the vmcore to upload", + ) + parser.add_argument( + "--core-directory", + type=Path, + help=f"where to store vmcores (default: {str(CORE_DIR)})", + ) + parser.add_argument( + "--par-url", + type=str, + default=os.environ.get("OCI_PAR_URL"), + help="pre authenticated request URL", + ) + parser.add_argument( + "--prefix", + type=str, + default=None, + help=f"prefix for vmcores in object storage (default: {CORE_PFX})", + ) + parser.add_argument( + "--delete-orphan", + action="store_true", + help="delete any files which are not listed on block storage", + ) + args = parser.parse_args() + if args.core_directory: + CORE_DIR = args.core_directory.absolute() + if args.prefix: + CORE_PFX = args.core_directory.absolute() + if not args.par_url: + sys.exit("error: either --par-url or $OCI_PAR_URL is required") + client = ParClient(args.par_url) + if args.action == "download": + download_all(client) + if args.delete_orphan: + delete_orphans(client) + elif args.action == "upload": + if not args.upload_core: + sys.exit("error: --upload-core is required for upload operation") + upload_all(client, args.upload_core) + + +if __name__ == "__main__": + main() diff --git a/testing/vmcore/test.py b/testing/vmcore/test.py new file mode 100644 index 00000000..6ebcec7b --- /dev/null +++ b/testing/vmcore/test.py @@ -0,0 +1,172 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +""" +Run tests in parallel against vmcores +""" +import argparse +import fnmatch +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import List +from typing import Tuple + +from drgn_tools.debuginfo import CtfCompatibility +from drgn_tools.debuginfo import KernelVersion +from testing.util import combine_junit_xml +from testing.vmcore.manage import CORE_DIR + + +def _test_job( + core_name: str, cmd: List[str], xml: str +) -> Tuple[str, bool, ET.ElementTree]: + # Runs the test silently, but prints the stdout/stderr on failure. + with NamedTemporaryFile("w+t") as f: + print(f"Begin testing {core_name}") + start = time.time() + res = subprocess.run(cmd, stdout=f, stderr=f) + if res.returncode != 0: + print(f"=== FAILURE: {core_name} ===") + f.seek(0) + sys.stdout.write(f.read()) + runtime = time.time() - start + print(f"Completed testing {core_name} in {runtime:.1f}") + run_data = ET.parse(xml) + return (core_name, res.returncode == 0, run_data) + + +def _skip_ctf(ctf: bool, uname: str) -> bool: + if ctf: + host_ol = 9 # OL8 or 9 work here, tests aren't supported for OL7 + kver = KernelVersion.parse(uname) + compat = CtfCompatibility.get(kver, host_ol) + # Skip test when CTF is fully unsupported, or when it would require a + # /proc/kallsyms. + return compat in ( + CtfCompatibility.NO, + CtfCompatibility.LIMITED_PROC, + ) + return False # don't skip when non-CTF + + +def test( + vmcore_list: List[str], + ctf: bool = False, + parallel: int = 1, +) -> None: + def should_run_vmcore(name: str) -> bool: + if not vmcore_list: + return True + for pat in vmcore_list: + if fnmatch.fnmatch(name, pat): + return True + return False + + failed = [] + passed = [] + skipped = [] + xml = None + + with ExitStack() as es: + pool = es.enter_context(ThreadPoolExecutor(max_workers=parallel)) + futures = [] + for path in CORE_DIR.iterdir(): + core_name = path.name + if not should_run_vmcore(core_name): + continue + uname = (path / "UTS_RELEASE").read_text().strip() + if _skip_ctf(ctf, uname): + skipped.append(core_name) + continue + xml_run = es.enter_context( + NamedTemporaryFile("w", suffix=".xml", delete=False) + ) + xml_run.close() # not deleted until context is ended + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/", + f"--vmcore={core_name}", + f"--vmcore-dir={str(CORE_DIR)}", + f"--junitxml={xml_run.name}", + "-o", + "junit_logging=all", + ] + if ctf: + if not (path / "vmlinux.ctfa").is_file(): + skipped.append(core_name) + continue + cmd.append("--ctf") + futures.append( + pool.submit(_test_job, core_name, cmd, xml_run.name) + ) + + for future in futures: + core_name, test_passed, run_data = future.result() + xml = combine_junit_xml(xml, run_data) + if test_passed: + passed.append(core_name) + else: + failed.append(core_name) + + if xml is not None: + xml.write("vmcore.xml") + print("Complete test logs: vmcore.xml") + print("Vmcore Test Summary -- Passed:") + print("\n".join(f"- {n}" for n in passed)) + if skipped: + print("Vmcore Test Summary -- Skipped (missing CTF):") + print("\n".join(f"- {n}" for n in skipped)) + if failed: + print("Vmcore Test Summary -- FAILED:") + print("\n".join(f"- {n}" for n in failed)) + sys.exit(1) + + +def main(): + global CORE_DIR + parser = argparse.ArgumentParser( + description="manages drgn-tools vmcores", + ) + parser.add_argument( + "--core-directory", type=Path, help="where to store vmcores" + ) + parser.add_argument( + "--vmcore", + action="append", + default=[], + help="only run tests on the given vmcore(s). you can use this " + "multiple times to specify multiple vmcore names. You can also " + "use fnmmatch patterns to specify several cores at once.", + ) + parser.add_argument( + "--ctf", + action="store_true", + help="Use CTF debuginfo for tests rather than DWARF (skips vmcores " + "without a vmlinux.ctfa file)", + ) + parser.add_argument( + "--parallel", + "-j", + type=int, + default=1, + help="Run the tests in parallel with the given number of threads", + ) + args = parser.parse_args() + if args.core_directory: + CORE_DIR = args.core_directory.absolute() + test( + args.vmcore, + ctf=args.ctf, + parallel=args.parallel, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index d1a646e3..dcb9e788 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,9 @@ def prog() -> drgn.Program: try: from drgn.helpers.linux.ctf import load_ctf + # CTF_FILE may be None here, in which case the default CTF file + # location is used (similar to below, where default DWARF debuginfo + # is loaded if we don't have a path). load_ctf(p, path=CTF_FILE) p.cache["using_ctf"] = True except ModuleNotFoundError: @@ -133,8 +136,12 @@ def pytest_configure(config): CORE_DIR = Path(core_dir) vmcore = config.getoption("vmcore") CTF = config.getoption("ctf") - debuginfo_kind = "DWARF" + debuginfo_kind = "CTF" if CTF else "DWARF" if vmcore: + # With vmcore tests, we need to manually find the debuginfo alongside + # the vmcore in the same directory. For heavyvm or litevm tests, the + # debuginfo is installed to the default locations, so we don't need any + # logic for them. VMCORE_NAME = vmcore vmcore_dir = CORE_DIR / vmcore vmcore_file = vmcore_dir / "vmcore" @@ -152,7 +159,6 @@ def pytest_configure(config): returncode=1, ) CTF_FILE = str(ctf_file) - debuginfo_kind = "CTF" else: vmlinux_file = vmcore_dir / "vmlinux" if not vmcore_file.is_file() or not vmlinux_file.is_file(): @@ -168,7 +174,7 @@ def pytest_configure(config): sys.version_info.major, sys.version_info.minor, sys.version_info.micro, - f"vmcore {vmcore}" if vmcore else "live", + f"vmcore {vmcore}" if vmcore else f"live {os.uname().release}", debuginfo_kind, ) if CTF: diff --git a/tox.ini b/tox.ini index 87418dbd..8a2da44e 100644 --- a/tox.ini +++ b/tox.ini @@ -10,9 +10,8 @@ skip_missing_interpreters = true sitepackages = true deps = -rrequirements-dev.txt - pytest-cov commands = - python -m pytest --cov=drgn_tools --cov=tests --junitxml=test.xml -o junit_logging=all {posargs} + python -m pytest --junitxml=test.xml -o junit_logging=all {posargs} passenv = DRGNTOOLS_*, GITLAB_CI, GITHUB_ACTIONS [testenv:docs]