From e80367244a87869182095b182638f9b1358fc341 Mon Sep 17 00:00:00 2001 From: Mitch Negus <21086604+mitchnegus@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:38:10 -0700 Subject: [PATCH] Provide CI testing support (#8) This provides a starting point for converting our GitLab CI testing infrastructure to the GitHub Actions workflow. ### Workflow Upgrades To avoid duplication in the workflow scripts, I've created a new set of [composite actions](https://docs.github.com/en/actions/sharing-automations/creating-actions/creating-a-composite-action) to install minimega, discovery, FIREWHEEL, and tox. We should hopefully be able to use these composite actions to dramatically simplify some of the CI scripts we had before. I've also upgraded the versions of referenced reusable versions where possible. ### Test Enhancements The unit tests have been upgraded to provide increased utility and work better through the FIREWHEEL interface: - Running the `firewheel test unit` helper now returns the appropriate exit code returned by pytest, which was previously unintentionally suppressed. - A new marker includes or excludes tests which depend on model components. - In service of that functionality, the tests now also make use of [pytest's built-in `-m` option](https://docs.pytest.org/en/stable/example/markers.html#marking-test-functions-and-selecting-them-for-a-run) to select marked tests for inclusion/exclusion rather than adding custom options. ### Bug Fixes This PR also fixes a few bugs in the current implementation: - This fixes a testing error identified by testing the FIREWHEEL deployment in an fresh installation (a hardcoded path in the `test_cli_completion.py` script). - Arguments passed to the helpers were not parsed correctly, ignoring quoted inputs and just splitting on spaces. This changes the helpers to use `shlex` for proper parsing. --- .github/actions/prepare-discovery/action.yml | 15 +++++ .github/actions/prepare-firewheel/action.yml | 33 ++++++++++ .github/actions/prepare-minimega/action.yml | 29 +++++++++ .github/actions/prepare-tox/action.yml | 15 +++++ .github/workflows/documentation.yml | 13 ++-- .github/workflows/linting.yml | 10 ++- .github/workflows/testing.yml | 49 ++++++++++++++ pyproject.toml | 19 +++--- src/firewheel/cli/firewheel_cli.py | 3 +- src/firewheel/cli/helpers/test/unit | 2 +- src/firewheel/cli/init_firewheel.py | 8 +-- src/firewheel/cli/utils.py | 3 +- src/firewheel/tests/conftest.py | 65 +++---------------- .../tests/unit/cli/test_cli_completion.py | 8 ++- src/firewheel/tests/unit/conftest.py | 5 ++ .../tests/unit/control/test_performance.py | 2 + tox.ini | 4 -- 17 files changed, 192 insertions(+), 91 deletions(-) create mode 100644 .github/actions/prepare-discovery/action.yml create mode 100644 .github/actions/prepare-firewheel/action.yml create mode 100644 .github/actions/prepare-minimega/action.yml create mode 100644 .github/actions/prepare-tox/action.yml create mode 100644 .github/workflows/testing.yml create mode 100644 src/firewheel/tests/unit/conftest.py diff --git a/.github/actions/prepare-discovery/action.yml b/.github/actions/prepare-discovery/action.yml new file mode 100644 index 0000000..f759651 --- /dev/null +++ b/.github/actions/prepare-discovery/action.yml @@ -0,0 +1,15 @@ +# A composite action to install discovery + +name: Prepare discovery + +description: Install discovery + +runs: + using: composite + steps: + - name: Install discovery + run: | + wget https://github.com/mitchnegus/minimega-discovery/releases/download/firewheel-debian_faed761/discovery.deb + sudo dpkg -i discovery.deb + sudo chown -R $USER:minimega /opt/discovery + shell: bash diff --git a/.github/actions/prepare-firewheel/action.yml b/.github/actions/prepare-firewheel/action.yml new file mode 100644 index 0000000..7545f06 --- /dev/null +++ b/.github/actions/prepare-firewheel/action.yml @@ -0,0 +1,33 @@ +# A composite action to install, configure, and initialize FIREWHEEL. +# This action assumes that minimega and discovery have already been installed. + +name: Prepare FIREWHEEL + +description: Install, configure, and initialize FIREWHEEL + +runs: + using: composite + steps: + - name: Install FIREWHEEL + run: | + pip install --upgrade pip + pip install . + sudo ln -s $(which firewheel) /usr/local/bin/firewheel + ssh-keygen -t rsa -f "$HOME/.ssh/id_rsa" -N "" + ssh-keyscan -t rsa $(hostname) >> $HOME/.ssh/known_hosts + cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys + shell: bash + - name: Configure FIREWHEEL + run: | + firewheel config set -s cluster.compute $(hostname) + firewheel config set -s cluster.control $(hostname) + firewheel config set -s grpc.hostname $GRPC_HOSTNAME + firewheel config set -s minimega.experiment_interface $EXPERIMENT_INTERFACE + firewheel config set -s logging.root_dir $LOG_DIR + shell: bash + - name: Initialize FIREWHEEL + run: | + firewheel init + firewheel sync # will produce `chgrp` errors (but permissions are sufficient) + firewheel start + shell: bash diff --git a/.github/actions/prepare-minimega/action.yml b/.github/actions/prepare-minimega/action.yml new file mode 100644 index 0000000..0a061e5 --- /dev/null +++ b/.github/actions/prepare-minimega/action.yml @@ -0,0 +1,29 @@ +# A composite action to install and initialize minimega + +name: Prepare minimega + +description: Install and initialize minimega + +runs: + using: composite + steps: + - name: Install minimega + run: | + wget https://github.com/sandia-minimega/minimega/releases/download/2.9/minimega-2.9.deb + sudo dpkg -i minimega-2.9.deb + sudo chown -R $USER:minimega $MM_INSTALL_DIR + sudo ln -s $MM_INSTALL_DIR/bin/minimega /usr/local/bin/minimega + sudo ln -s $MM_INSTALL_DIR/bin/minimega /usr/local/bin/mm + shell: bash + - name: Initialize minimega + run: | + echo -n "" | sudo $MM_INSTALL_DIR/misc/daemon/minimega.init install + sudo mkdir -p $(dirname $MINIMEGA_CONFIG) + sudo sed -i "s|MINIMEGA_DIR=\"/opt/minimega/\"|MINIMEGA_DIR=\"$MM_INSTALL_DIR/\"|g" $MINIMEGA_CONFIG + sudo sed -i "s|MM_RUN_PATH=\"/tmp/minimega\"|MM_RUN_PATH=\"$MM_BASE/\"|g" $MINIMEGA_CONFIG + sudo sed -i "s|MM_MESH_DEGREE=0|MM_MESH_DEGREE=1|g" $MINIMEGA_CONFIG + sudo sed -i "s|MM_LOG_LEVEL=\"error\"|MM_LOG_LEVEL=\"debug\"|g" $MINIMEGA_CONFIG + sudo sed -i "s|MM_LOG_FILE=\"/tmp/minimega.log\"|MM_LOG_FILE=\"$LOG_DIR/minimega.log\"|g" $MINIMEGA_CONFIG + sudo systemctl restart minimega + sudo chown -R $USER:minimega $MM_BASE $LOG_DIR + shell: bash diff --git a/.github/actions/prepare-tox/action.yml b/.github/actions/prepare-tox/action.yml new file mode 100644 index 0000000..46174ff --- /dev/null +++ b/.github/actions/prepare-tox/action.yml @@ -0,0 +1,15 @@ +# A composite action to prepare tox-based workflows + +name: Prepare tox + +description: Prepare tox-based workflows + +runs: + using: composite + steps: + - name: Ensure upgraded pip + run: pip install --upgrade pip + shell: bash + - name: Install tox + run: pip install tox + shell: bash diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3aef287..240453b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,5 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +# This workflow will build and deploy the FIREWHEEL documentation name: Documentation @@ -34,13 +33,11 @@ jobs: - name: Setup Pages uses: actions/configure-pages@v5 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install tox + - name: Prepare to use tox-based environments + uses: ./.github/actions/prepare-tox - name: Build Documentation run: | tox -e dependencies,docs @@ -60,4 +57,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index abd0f5e..6d06019 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies and lint with a variety of Python versions # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: Linting @@ -23,13 +23,11 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install tox + - name: Prepare to use tox-based environments + uses: ./.github/actions/prepare-tox - name: Lint code run: | tox -e lint diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 0000000..fb5d57d --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,49 @@ +# This workflow will install Python dependencies and run tests with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Testing + +on: + push: + branches: [ "*" ] + pull_request: + branches: [ "main" ] + +env: + LOG_DIR: /var/log/firewheel + MINIMEGA_CONFIG: /etc/minimega/minimega.conf + # Set the FIREWHEEL environment variables + EXPERIMENT_INTERFACE: lo + MM_BASE: /tmp/minimega + MM_INSTALL_DIR: /opt/minimega + GRPC_HOSTNAME: localhost + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y tar net-tools procps uml-utilities \ + openvswitch-switch qemu-kvm qemu-utils dnsmasq \ + ntfs-3g iproute2 libpcap-dev + - name: Prepare minimega + uses: ./.github/actions/prepare-minimega + - name: Prepare discovery + uses: ./.github/actions/prepare-discovery + - name: Prepare FIREWHEEL + uses: ./.github/actions/prepare-firewheel + - name: Run unit tests + run: | + firewheel test unit -m 'not long and not mcs' \ + --cov --cov-report=term --cov-fail-under=60 diff --git a/pyproject.toml b/pyproject.toml index 95c8b1b..8317260 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,28 +55,30 @@ classifiers = [ ] dependencies = [ "minimega==2.9", + "ClusterShell<=1.9.2", "colorama<=0.4.6", + "coverage<=7.6.10", "grpcio>=1.49.0,<=1.67.0", "grpcio-tools>=1.49.0,<=1.69.0", + "importlib_metadata>=3.6,<=8.5.0", "Jinja2>=3.1.2,<=3.1.5", "netaddr<=1.3.0,>=0.7.0", "networkx>=2.3,<=3.4.2", "protobuf>=5.0.0,<=5.29.3", - "ClusterShell<=1.9.2", "pytest<=8.3.4", + "pytest-cov<=6.0.0", "python-dotenv<=1.0.1", "PyYAML<=6.0.2", "qemu.qmp==0.0.3", - "rich>=13.6.0,<13.10", "requests>=2.22.0,<=2.32.3", - "importlib_metadata>=3.6,<=8.5.0", + "rich>=13.6.0,<13.10", ] [project.optional-dependencies] -test = [ - "tox<=4.23.2", - "coverage<=7.6.10", - "pytest-cov<=6.0.0", +mcs = [ + "firewheel-repo-base", + "firewheel-repo-linux", + "firewheel-repo-vyos", ] format = [ "ruff==0.9.2", # Linting/formatting @@ -95,8 +97,9 @@ docs = [ "sphinx-design", ] dev = [ - "firewheel[test,format,docs]", + "firewheel[mcs,format,docs]", "pre-commit", + "tox~=4.0", ] [project.urls] # Optional diff --git a/src/firewheel/cli/firewheel_cli.py b/src/firewheel/cli/firewheel_cli.py index 3be3589..af6c240 100755 --- a/src/firewheel/cli/firewheel_cli.py +++ b/src/firewheel/cli/firewheel_cli.py @@ -3,6 +3,7 @@ import os import cmd import sys +import shlex import logging import textwrap from math import floor @@ -1199,7 +1200,7 @@ def complete_help(self, text, line, _begidx, _endidx): def main(): # pragma: no cover """Provide an entry point to the FIREWHEEL CLI.""" if len(sys.argv) > 1: - argstr = " ".join(sys.argv[1:]) + argstr = shlex.join(sys.argv[1:]) try: sys.exit(FirewheelCLI().onecmd(argstr)) except KeyboardInterrupt: diff --git a/src/firewheel/cli/helpers/test/unit b/src/firewheel/cli/helpers/test/unit index 8492c1a..c465382 100644 --- a/src/firewheel/cli/helpers/test/unit +++ b/src/firewheel/cli/helpers/test/unit @@ -21,5 +21,5 @@ from firewheel import FIREWHEEL_PACKAGE_DIR if __name__ == "__main__": test_dir = FIREWHEEL_PACKAGE_DIR / "tests" / "unit" - pytest.main([str(test_dir), *sys.argv[1:]]) + sys.exit(pytest.main([*sys.argv[1:], str(test_dir)])) DONE diff --git a/src/firewheel/cli/init_firewheel.py b/src/firewheel/cli/init_firewheel.py index 619d408..0e61c17 100644 --- a/src/firewheel/cli/init_firewheel.py +++ b/src/firewheel/cli/init_firewheel.py @@ -118,16 +118,16 @@ def _check_minimega_socket(self): Returns: bool: False if minimega is not running, True otherwise. """ - status = False try: minimegaAPI() - status = True - return True except (RuntimeError, TimeoutError): - return False + status = False + else: + status = True finally: success_str = self._get_success_str(status) print(f"Checking minimega service status: {success_str}") + return status def _get_minimega_install_dir(self): # We should check that the minimega bin is in the expected location. diff --git a/src/firewheel/cli/utils.py b/src/firewheel/cli/utils.py index cb3b9dd..1d1580f 100644 --- a/src/firewheel/cli/utils.py +++ b/src/firewheel/cli/utils.py @@ -1,5 +1,6 @@ import os import sys +import shlex from pathlib import Path from rich.table import Table @@ -78,7 +79,7 @@ def parse_to_helper(args, helpers_dict): InvalidHelperTypeError: If a Helper group was specified rather than a Helper. """ # Check for the Helper name in the Helpers dict. - args = args.split() + args = shlex.split(args) if args[0] not in helpers_dict: raise HelperNotFoundError("Unable to find Helper") # Check the type of the Helper entry. diff --git a/src/firewheel/tests/conftest.py b/src/firewheel/tests/conftest.py index 7a16abe..6c11710 100644 --- a/src/firewheel/tests/conftest.py +++ b/src/firewheel/tests/conftest.py @@ -10,33 +10,9 @@ `. """ -from typing import List - import pytest -def pytest_addoption(parser: pytest.Parser) -> None: - """ - Register argparse-style options for running tests. - - Register argparse-style options and ini-style config values, - called once at the beginning of a test run. This is an - `initialization hook - ` - provided by :py:mod:`pytest`. - - Args: - parser (pytest.Parser): The parser that will received added - options. - """ - parser.addoption( - "--quick", - action="store_true", - default=False, - help="exclude tests marked as long", - ) - - def pytest_configure(config: pytest.Config) -> None: """ Enable this conftest file to perform initial configuration. @@ -50,39 +26,16 @@ def pytest_configure(config: pytest.Config) -> None: they are imported. This specific hook adds custom markers to the test suite, such as a - ``long`` marker to indicate long-running tests. - - Args: - config (pytest.Config): The pytest config object. - """ - # As a heuristic, mark tests that take more than 10 seconds as "long" - config.addinivalue_line("markers", "long: mark test as long running") - - -def pytest_collection_modifyitems( - session: pytest.Session, # noqa: ARG001 - config: pytest.Config, - items: List[pytest.Item], -) -> None: - """ - Modify the set of tests/items collected by :py:mod:`pytest`. - - This is a `collection hook - ` - provided by :py:mod:`pytest` to modify the set of tests or items - collected by the test runner. The hook is called after collection - has been performed and it may filter or re-order the items in-place. + ``long`` marker to indicate long-running tests (more than 10 seconds + duration) and the ``mcs`` marker to indicate tests that require + model components beyond the base FIREWHEEL package. Args: - session (pytest.Session): The pytest session object. config (pytest.Config): The pytest config object. - items (list): A list of item objects """ - # Use the custom `--quick` option with the `long` marker to skip long running tests - if config.getoption("--quick"): - skip_long = pytest.mark.skip( - reason="--quick option excludes long running tests" - ) - for item in items: - if "long" in item.keywords: - item.add_marker(skip_long) + markers = [ + "long: mark test as long running", + "mcs: mark test as dependent on model components", + ] + for marker in markers: + config.addinivalue_line("markers", marker) diff --git a/src/firewheel/tests/unit/cli/test_cli_completion.py b/src/firewheel/tests/unit/cli/test_cli_completion.py index 58f5f32..e651228 100644 --- a/src/firewheel/tests/unit/cli/test_cli_completion.py +++ b/src/firewheel/tests/unit/cli/test_cli_completion.py @@ -3,6 +3,9 @@ from pathlib import Path from unittest.mock import Mock, patch, mock_open +import pytest + +from firewheel.config import config from firewheel.cli.completion import COMPLETION_SCRIPT_PATH from firewheel.cli.completion.actions import ( _keyboard_interruptable, @@ -64,6 +67,7 @@ def test_get_model_component_names(self, mock_iterator_cls, mock_stdout): printed_mc_names = printed_output.strip().split(" ") self.assertCountEqual(printed_mc_names, mock_mc_names) + @pytest.mark.mcs @patch("sys.stdout", new_callable=io.StringIO) def test_get_total_model_components_size(self, mock_stdout): # Check that the size is nonzero (some MCs were found) @@ -95,8 +99,8 @@ def test_populate_template(self): mock_handle.write.assert_called_once() script_content = mock_handle.write.call_args.args[0] filled_placeholders = [ - 'fw_venv="/opt/firewheel/fwpy"', - 'python_bin="python3"', + f"fw_venv=\"{config['python']['venv']}\"", + f"python_bin=\"{config['python']['bin']}\"", ] assert all(_ in script_content for _ in filled_placeholders) diff --git a/src/firewheel/tests/unit/conftest.py b/src/firewheel/tests/unit/conftest.py new file mode 100644 index 0000000..1fb36ad --- /dev/null +++ b/src/firewheel/tests/unit/conftest.py @@ -0,0 +1,5 @@ +# Typically, conftest imports are not allowed, but since the test suites +# are part of the FIREWHEEL package, the conftest file is a module and +# is importable. This allows the plugin hooks to be loaded when running +# the unit test set independently using the FIREWHEEL helper. +from firewheel.tests.conftest import pytest_configure # noqa: F401 diff --git a/src/firewheel/tests/unit/control/test_performance.py b/src/firewheel/tests/unit/control/test_performance.py index 414dd76..1548137 100644 --- a/src/firewheel/tests/unit/control/test_performance.py +++ b/src/firewheel/tests/unit/control/test_performance.py @@ -104,6 +104,7 @@ def get_stats(self, stats, number=5): ) return dict(itertools.islice(sorted_d.items(), number)) + @pytest.mark.mcs @pytest.mark.parametrize( ["num_runs", "avg_threshold"], [ @@ -137,6 +138,7 @@ def test_average_performance_building_dep_graph( f"Actual: {avg_time} seconds" ) + @pytest.mark.mcs @pytest.mark.long @patch("sys.stdout", new_callable=io.StringIO) def test_individual_experiment_graph(self, mock_stdout, cache_vms): diff --git a/tox.ini b/tox.ini index 9fa2abc..dfd038d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,6 @@ skip_missing_interpreters = true passenv = PYTHONWARNINGS PYTHONDEVMODE - PIP_DISABLE_PIP_VERSION_CHECK - PIP_INDEX_URL - PIP_EXTRA_INDEX_URL -extras = test usedevelop = true commands = pytest {posargs} src/firewheel/tests/