diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f5676750..82aca312 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -101,6 +101,7 @@ jobs: - extra_metadata - elfdeps - prebuilt_wheel_hook + - lint_requirements os: - ubuntu-latest - macos-latest diff --git a/.mergify.yml b/.mergify.yml index b570b97d..b39493f6 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -48,6 +48,7 @@ pull_request_rules: - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest) - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest) - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest) + - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest) - check-success=e2e (3.12, 1.75, bootstrap, macos-latest) - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest) - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest) @@ -92,6 +93,8 @@ pull_request_rules: - check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest) - check-success=e2e (3.12, 1.75, rust_vendor, macos-latest) - check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest) + - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest) + - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest) - "-draft" # At least 1 reviewer diff --git a/e2e/test_lint_requirements.sh b/e2e/test_lint_requirements.sh new file mode 100755 index 00000000..7698da80 --- /dev/null +++ b/e2e/test_lint_requirements.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# This script acts as an e2e test for lint_requirements command of fromager + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +pass=true + +# Test to demonstrate that command works as expected when input files are valid + +if ! fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then + echo "the input files should have been recognized as correctly formatted" 1>&2 + pass=false +fi; + +# Test to demonstrate failing of command due to missing input args + +if fromager lint-requirements; then + echo "missing input files should have been recognized by the command" 1>&2 + pass=false +fi; + +# Test to demonstrate that command reports error for invalid / bad input files + +if fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/invalid-requirements.txt"; then + echo "invalid input files should have been recognized by the command" 1>&2 + pass=false +fi; + +$pass diff --git a/e2e/validate_inputs/constraints.txt b/e2e/validate_inputs/constraints.txt new file mode 100644 index 00000000..18662d43 --- /dev/null +++ b/e2e/validate_inputs/constraints.txt @@ -0,0 +1,5 @@ +# This requirements.txt is used for testing lint_requirements +# command of fromager and contains examples of packages + +gast==0.5.5 +torch==2.3.1 diff --git a/e2e/validate_inputs/invalid-requirements.txt b/e2e/validate_inputs/invalid-requirements.txt new file mode 100644 index 00000000..cbfb06de --- /dev/null +++ b/e2e/validate_inputs/invalid-requirements.txt @@ -0,0 +1,5 @@ +# This requirements file is invalid on purpose to test the lint-requirements +# command of fromager. Do not attempt to fix this file + +foo== +bar<= diff --git a/e2e/validate_inputs/requirements.txt b/e2e/validate_inputs/requirements.txt new file mode 100644 index 00000000..ffdab955 --- /dev/null +++ b/e2e/validate_inputs/requirements.txt @@ -0,0 +1,5 @@ +# This requirements.txt is used for testing lint_requirements +# command of fromager and contains examples of packages +stevedore==5.2.0 +kfp==2.11.0 + diff --git a/src/fromager/commands/__init__.py b/src/fromager/commands/__init__.py index 90775aec..acd65403 100644 --- a/src/fromager/commands/__init__.py +++ b/src/fromager/commands/__init__.py @@ -6,6 +6,7 @@ download_sequence, graph, lint, + lint_requirements, list_overrides, migrate_config, server, @@ -26,4 +27,5 @@ download_sequence.download_sequence, server.wheel_server, migrate_config.migrate_config, + lint_requirements.lint_requirements, ] diff --git a/src/fromager/commands/lint_requirements.py b/src/fromager/commands/lint_requirements.py new file mode 100644 index 00000000..106908ef --- /dev/null +++ b/src/fromager/commands/lint_requirements.py @@ -0,0 +1,39 @@ +import logging +import sys + +import click +from packaging.requirements import InvalidRequirement, Requirement + +from fromager import requirements_file + +logger = logging.getLogger(__name__) + + +@click.command() +@click.argument( + "input_files_path", nargs=-1, required=True, type=click.Path(exists=False) +) +def lint_requirements(input_files_path: list[click.Path]) -> None: + """ + Command to lint the constraints.txt and requirements.txt files + This command takes a single wildcard path string for constraints.txt and requirements.txt. + It checks the formatting of these files and reports issues if found. + """ + + if len(input_files_path) == 0: + logger.error("no constraints.txt or requirements.txt found in given paths") + sys.exit(1) + + flag = True + + for path in input_files_path: + parsed_lines = requirements_file.parse_requirements_file(str(path)) + for line in parsed_lines: + try: + Requirement(line) + except InvalidRequirement as err: + logger.error(f"{path}: {line}: {err}") + flag = False + + if not flag: + sys.exit(1)