[v1] CI Improvements

.github/workflows/litevm.yml

@@ -20,49 +20,49 @@ jobs:
       - name: Run pre-commit hooks
         run: pre-commit run --all-files --show-diff-on-failure
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        # OL7 and OL8 use 3.6. OL9 uses 3.9. Also test on the latest.
-        python-minor: ["6", "9", "12"]
+        # Our minimum supported version of Python is 3.6, used by Oracle Linux 7
+        # & 8. Ideally we would run tests on that, along with Python 3.9 for
+        # Oracle Linux 9, and maybe Python 3.12 for an upcoming Oracle Linux 10.
+        # However, practicality rules here. The binutils provided in Ubuntu
+        # 20.04 is not recent enough for our libctf usage, but 20.04 is the only
+        # remaining Ubuntu image with Python 3.6 available in Github actions.
+        # So we have to eliminate Python 3.6 from our test matrix. Other CI
+        # tests do run on Python 3.6, and there is the "vermin" pre-commit hook
+        # which detects code incompatible with 3.6.
+        python-minor: ["9", "12"]
       fail-fast: false
-    env:
-      # Manually set the tox environment list to the Python version.
-      # As a result, we should set "skip missing interpreters" to false,
-      # so that we fail if the test doesn't run.
-      TOX_OVERRIDE: "tox.envlist=py3${{ matrix.python-minor }}"
-      TOX_SKIP_MISSING_INTERPRETERS: "false"
     steps:
       - uses: actions/checkout@v4
-      # If we rely on using the same Python version for tox as we do for the
-      # testing version of Python, we end up getting different versions of tox
-      # with different behavior. Let's setup a well-defined python version for
-      # tox and use that. We need to place this one before the regular python
-      # setup so that the regular python takes precedence.
-      - name: Set up Python for Tox
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.${{ matrix.python-minor }}
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install qemu-kvm zstd gzip bzip2 cpio busybox-static fio
+          sudo apt-get install qemu-kvm zstd gzip bzip2 cpio busybox-static fio \
+                               autoconf automake check gcc git liblzma-dev \
+                               libelf-dev libdw-dev libtool make pkgconf zlib1g-dev \
+                               binutils-dev
           sudo wget -O /usr/bin/rpm2cpio https://github.com/rpm-software-management/rpm/raw/rpm-4.19.0-release/scripts/rpm2cpio.sh
           echo '0403da24a797ccfa0cfd37bd4a6d6049370b9773e558da6173ae6ad25f97a428 /usr/bin/rpm2cpio' | sha256sum -c -
           sudo chmod 755 /usr/bin/rpm2cpio
       - name: Setup test environment
         # Pinned virtualenv and tox are for the last versions which support
         # detecting Python 3.6 and running tests on it.
         run: |
-          python3.12 -m pip install --user --break-system-packages 'virtualenv<20.22.0' 'tox<4.5.0'
-          sed -i 's/sitepackages = true/sitepackages = false/' tox.ini
-          tox list
-          tox --notest
-          tox -e runner --notest
+          python -m venv venv
+          venv/bin/pip install -r testing/requirements-litevm.txt
+          venv/bin/pip install setuptools
+      - name: Build and install drgn with CTF support
+        run: |
+          cd ..
+          git clone https://github.com/brenns10/drgn -b ctf_0.0.29
+          cd drgn
+          ../drgn-tools/venv/bin/python setup.py install
       - name: Run tests
         run: |
-          tox -e runner -- python -m testing.litevm.vm --delete-after-test --python-version 3${{ matrix.python-minor }}
+          venv/bin/python -m testing.litevm.vm --delete-after-test --with-ctf

.gitlab-ci.yml

@@ -4,11 +4,11 @@ vmtest:
   # time.
   resource_group: VM
   script:
-    - rm -rf .tox
+    - python -m venv venv --system-site-packages
+    - venv/bin/pip install -r testing/requirements-heavyvm.txt
     - git archive HEAD -o archive.tar.gz
-    - tox -e runner --notest
     - mkdir -p tmp/overlays tmp/info
-    - tox -e runner -- python -m testing.heavyvm.runner --image-dir /var/drgn-tools/images --vm-info-dir tmp/info --overlay-dir tmp/overlays --tarball archive.tar.gz
+    - venv/bin/python -m testing.heavyvm.runner --image-dir /var/drgn-tools/images --vm-info-dir tmp/info --overlay-dir tmp/overlays --tarball archive.tar.gz
   artifacts:
     when: always
     paths:
@@ -18,9 +18,9 @@ vmtest:
 
 vmcore DWARF:
   script:
-    - rm -rf .tox
-    - tox -e runner --notest
-    - tox -e runner -- python -m testing.vmcore test -e py39 -j 4 --core-directory /var/drgn-tools/vmcores
+    - python -m venv venv --system-site-packages
+    - venv/bin/pip install -r testing/requirements-vmcore.txt
+    - venv/bin/python -m testing.vmcore.test -j 4 --core-directory /var/drgn-tools/vmcores
   artifacts:
     when: always
     paths:
@@ -30,9 +30,9 @@ vmcore DWARF:
 
 vmcore CTF:
   script:
-    - rm -rf .tox
-    - tox -e runner --notest
-    - tox -e runner -- python -m testing.vmcore test -e py39 -j 4 --ctf --core-directory /var/drgn-tools/vmcores
+    - python -m venv venv --system-site-packages
+    - venv/bin/pip install -r testing/requirements-vmcore.txt
+    - venv/bin/python -m testing.vmcore.test -j 4 --ctf --core-directory /var/drgn-tools/vmcores
   artifacts:
     when: always
     paths:

.pre-commit-config.yaml

@@ -41,5 +41,5 @@ repos:
     rev: 0.1.1
     hooks:
     -   id: copyright-notice
-        exclude: (man|doc)/.*
+        exclude: ((man|doc)/.*|.*requirements.*\.txt)
         args: [--notice=.header.txt]

Makefile

@@ -12,23 +12,23 @@ PYTHON ?= python3
 
 .PHONY: litevm-test
 litevm-test:
-	tox --notest
-	-tox -e runner -- python -m testing.litevm.vm
+	$(PYTHON) -m testing.litevm.vm
 
 
 .PHONY: vmcore-test
 vmcore-test:
-	-tox -e runner -- python -m testing.vmcore test
+	$(PYTHON) -m testing.vmcore test
 
 
+.PHONY: test
 test: litevm-test vmcore-test
 
 .PHONY: docs
 docs:
 	@$(PYTHON) -m tox -e docs
 
 drgn_tools/_version.py:
-	python setup.py -V
+	$(PYTHON) setup.py -V
 
 .PHONY: rsync
 rsync: drgn_tools/_version.py

drgn_tools/debuginfo.py

@@ -675,7 +675,9 @@ def get(
             return cls.NO
 
         # Prior to UEK4, it is untested and will not be tested.
-        if kver.uek_version < 4:
+        # UEK4 itself has broken CTF data (e.g. struct page) and this means that
+        # a large majority of helpers cannot function.
+        if kver.uek_version <= 4:
             return cls.NO
 
         # The OL7 libctf version does not support CTF generated for kernels on
@@ -705,9 +707,7 @@ def get(
             6: (2136, 312, 2),
             7: (3, 60, 2),
         }
-        if kver.uek_version == 4:
-            return cls.LIMITED_PROC
-        elif (
+        if (
             kver.uek_version < 8
             and kver.release_tuple < kallsyms_backport[kver.uek_version]
         ):

testing/README.md

@@ -207,13 +207,13 @@ manage your vmcores locally without needing to use OCI object storage.
 Assuming you have created the necessary directories and files, you can use the
 following to run tests:
 
-    python -m testing.vmcore test
+    python -m testing.vmcore.test
 
 If you've configured OCI and your `VMCORE_*` variables, then you can use the
 uploader to upload a specific vmcore, or download all vmcores.
 
-    python -m testing.vmcore upload --upload-core $name
-    python -m testing.vmcore downoad
+    python -m testing.vmcore.manage upload --upload-core $name
+    python -m testing.vmcore.manage downoad
 
 Vmcores
 -------

testing/heavyvm/runner.py

@@ -7,9 +7,9 @@
 import tempfile
 import time
 import typing as t
+import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from junitparser import JUnitXml
 from paramiko.client import AutoAddPolicy
 from paramiko.client import SSHClient
 
@@ -20,6 +20,7 @@
 from testing.util import BASE_DIR
 from testing.util import ci_section_end
 from testing.util import ci_section_start
+from testing.util import combine_junit_xml
 
 
 @dataclasses.dataclass
@@ -90,6 +91,7 @@ class TestRunner:
 
     _vms_up: bool
     _ssh: t.Dict[str, SSHClient]
+    _xml: t.Optional[ET.ElementTree]
 
     def _section_start(
         self, name: str, text: str, collapsed: bool = False
@@ -164,6 +166,7 @@ def __init__(
         self.overlay_dir = overlay_dir or self.image_dir
         self._vms_up = False
         self._ssh = {}
+        self._xml = None
         self.images = images or []
         if self.vm_info_file.exists():
             with self.vm_info_file.open() as f:
@@ -248,20 +251,19 @@ def run_cmd(self, cmd: str) -> None:
             print("Result:\n" + result)
         self._section_end("run_cmd")
 
-    def _get_result_xml(self, info: VmInfo) -> JUnitXml:
+    def _get_result_xml(self, info: VmInfo) -> ET.ElementTree:
         ssh_client = self._get_ssh(info)
         sftp = ssh_client.open_sftp()
-        # This XML contains an encoding declaration, and if we decode it, that
-        # will trigger an error in the xml module. Leave it as bytes, even
-        # though the JUnitXml.fromstring() function insists it takes a str.
-        xmldata = sftp.file("/root/test/test.xml").read()
-        return JUnitXml.fromstring(xmldata)
+        return ET.parse(sftp.file("/root/test/test.xml"))
 
-    def run_test(self, cmd: str) -> int:
+    def run_test(self, cmd: str, ctf: bool = False) -> int:
         fail_list = []
-        xml = JUnitXml()
         for vm in self.vms.values():
+            if vm.uek_version == 4 and ctf:
+                continue
             slug = f"ol{vm.ol_version[0]}uek{vm.uek_version}"
+            if ctf:
+                slug += "_CTF"
             self._section_start(
                 f"test_{slug}", f"Running test on {slug}", collapsed=True
             )
@@ -273,7 +275,7 @@ def run_test(self, cmd: str) -> int:
             else:
                 print("Failed.")
                 fail_list.append(vm.name)
-            xml += self._get_result_xml(vm)
+            self._xml = combine_junit_xml(self._xml, self._get_result_xml(vm))
             self._section_end(f"test_{slug}")
         if fail_list:
             print(
@@ -286,7 +288,8 @@ def run_test(self, cmd: str) -> int:
         output = Path("heavyvm.xml")
         if output.exists():
             output.unlink()
-        xml.write(str(output))
+        if self._xml is not None:
+            self._xml.write(str(output))
         return len(fail_list)
 
 
@@ -330,11 +333,14 @@ def main():
     )
     with r:
         r.copy_extract_files(args.tarball)
-        sys.exit(
-            r.run_test(
-                "cd /root/test && python3 -m pytest --junitxml=test.xml -o junit_logging=all tests"
-            )
+        test_cmd = (
+            "cd /root/test && "
+            "python3 -m pytest --junitxml test.xml -o junit_logging=all tests"
         )
+        fail_count = 0
+        fail_count += r.run_test(test_cmd)
+        fail_count += r.run_test(test_cmd + " --ctf", ctf=True)
+        sys.exit(fail_count)
 
 
 if __name__ == "__main__":