Add option to run CI on all kernel flavors, plus other improvements

.github/workflows/ci.yml

@@ -11,13 +11,23 @@ on:
         type: boolean
         default: false
         required: true
+      test_all_kernel_flavors:
+        description: "Run tests on all kernel flavors"
+        type: boolean
+        default: false
+        required: true
   workflow_call:
     inputs:
       test_all_python_versions:
         description: "Run tests on all Python versions"
         type: boolean
         default: false
         required: true
+      test_all_kernel_flavors:
+        description: "Run tests on all kernel flavors"
+        type: boolean
+        default: false
+        required: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -62,7 +72,7 @@ jobs:
         if: ${{ env.USE_PRE_COMMIT == '1' }}
         run: pre-commit run --all-files mypy
       - name: Build and test with ${{ matrix.cc }}
-        run: CONFIGURE_FLAGS="--enable-compiler-warnings=error" python setup.py test -K
+        run: CONFIGURE_FLAGS="--enable-compiler-warnings=error" python setup.py test -K ${{ inputs.test_all_kernel_flavors && '-F' || '' }}
 
   lint:
     runs-on: ubuntu-latest

.github/workflows/pull_request.yml

@@ -14,3 +14,4 @@ jobs:
     if: ${{ github.event.action != 'labeled' || github.event.label.name == 'test-all-python-versions' }}
     with:
       test_all_python_versions: ${{ contains(github.event.pull_request.labels.*.name, 'test-all-python-versions') }}
+      test_all_kernel_flavors: ${{ contains(github.event.pull_request.labels.*.name, 'test-all-kernel-flavors') }}

.github/workflows/vmtest-build.yml

@@ -41,3 +41,4 @@ jobs:
     uses: ./.github/workflows/ci.yml
     with:
       test_all_python_versions: true
+      test_all_kernel_flavors: true

setup.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import sysconfig
@@ -163,6 +164,12 @@ class test(Command):
             "run Linux kernel tests in a virtual machine on all supported kernels "
             f"({', '.join(SUPPORTED_KERNEL_VERSIONS)})",
         ),
+        (
+            "flavor=",
+            "f",
+            "when combined with -K, run Linux kernel tests on a specific flavor "
+            f"({', '.join(KERNEL_FLAVORS)}) instead of the default flavor",
+        ),
         (
             "all-kernel-flavors",
             "F",
@@ -184,14 +191,15 @@ class test(Command):
 
     def initialize_options(self):
         self.kernel = False
+        self.flavor = "default"
         self.all_kernel_flavors = False
         self.extra_kernels = ""
         self.vmtest_dir = None
 
     def finalize_options(self):
         self.kernels = [kernel for kernel in self.extra_kernels.split(",") if kernel]
         if self.kernel:
-            flavors = KERNEL_FLAVORS if self.all_kernel_flavors else ["default"]
+            flavors = KERNEL_FLAVORS if self.all_kernel_flavors else [self.flavor]
             self.kernels.extend(
                 kernel + ".*" + flavor
                 for kernel in SUPPORTED_KERNEL_VERSIONS
@@ -256,7 +264,9 @@ def run(self):
         from vmtest.config import ARCHITECTURES, Kernel, local_kernel
         from vmtest.download import DownloadCompiler, DownloadKernel, download_in_thread
 
-        if os.getenv("GITHUB_ACTIONS") == "true":
+        in_github_actions = os.getenv("GITHUB_ACTIONS") == "true"
+
+        if in_github_actions:
 
             @contextlib.contextmanager
             def github_workflow_group(title):
@@ -285,7 +295,16 @@ def github_workflow_group(title):
                         to_download.append(
                             DownloadKernel(ARCHITECTURES["x86_64"], pattern)
                         )
-            with download_in_thread(Path(self.vmtest_dir), to_download) as downloads:
+
+            # Downloading too many files before they can be used for testing runs the
+            # risk of filling up the limited disk space is Github Actions. Set a limit
+            # of no more than 5 files which can be downloaded ahead of time. This is a
+            # magic number which is inexact, but works well enough.
+            max_pending_kernels = 5 if in_github_actions else 0
+
+            with download_in_thread(
+                Path(self.vmtest_dir), to_download, max_pending_kernels
+            ) as downloads:
                 downloads_it = iter(downloads)
 
                 if to_download:
@@ -327,6 +346,12 @@ def github_workflow_group(title):
                         logger.info("Passed: %s", ", ".join(passed))
                     if failed:
                         logger.error("Failed: %s", ", ".join(failed))
+
+                    # Github Actions has limited disk space. Once tested, we
+                    # will not use the kernel again, so delete it.
+                    if in_github_actions:
+                        logger.info("Deleting kernel %s", kernel.release)
+                        shutil.rmtree(kernel.path)
         except urllib.error.HTTPError as e:
             if e.code == 403:
                 print(e, file=sys.stderr)

vmtest/__main__.py

@@ -5,6 +5,7 @@
 import os
 from pathlib import Path
 import shlex
+import shutil
 import subprocess
 import sys
 from typing import Dict, List, TextIO
@@ -243,7 +244,19 @@ def add_kernel(arch: Architecture, pattern: str) -> None:
 
     progress = _ProgressPrinter(sys.stderr)
 
-    with download_in_thread(args.directory, to_download) as downloads:
+    in_github_actions = os.getenv("GITHUB_ACTIONS") == "true"
+
+    # Downloading too many files before they can be used for testing runs the
+    # risk of filling up the limited disk space is Github Actions. Set a limit
+    # of no more than 5 files which can be downloaded ahead of time. This is a
+    # magic number which is inexact, but works well enough.
+    # Note that Github Actions does not run vmtest via this script currently,
+    # but may in the future.
+    max_pending_kernels = 5 if in_github_actions else 0
+
+    with download_in_thread(
+        args.directory, to_download, max_pending_kernels
+    ) as downloads:
         for arch in architectures:
             if arch is HOST_ARCHITECTURE:
                 subprocess.check_call(
@@ -337,4 +350,7 @@ def add_kernel(arch: Architecture, pattern: str) -> None:
             except LostVMError as e:
                 print("error:", e, file=sys.stderr)
                 status = -1
+
+            if in_github_actions:
+                shutil.rmtree(kernel.path)
             progress.update(kernel.arch.name, kernel.release, status == 0)

vmtest/download.py

@@ -284,9 +284,11 @@ def _download_thread(
 
 @contextmanager
 def download_in_thread(
-    download_dir: Path, downloads: Iterable[Download]
+    download_dir: Path, downloads: Iterable[Download], max_pending_kernels: int = 0
 ) -> Generator[Iterator[Downloaded], None, None]:
-    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue()
+    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue(
+        maxsize=max_pending_kernels
+    )
 
     def aux() -> Iterator[Downloaded]:
         while True: