[Feat]: Add lint-requirements command

Author: rd4398

.github/workflows/test.yaml

@@ -101,6 +101,7 @@ jobs:
           - extra_metadata
           - elfdeps
           - prebuilt_wheel_hook
+          - lint_requirements
         os:
           - ubuntu-latest
           - macos-latest

.mergify.yml

@@ -48,6 +48,7 @@ pull_request_rules:
           - check-success=e2e (3.11, 1.75, prebuilt_wheels_alt_server, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, report_missing_dependency, ubuntu-latest)
           - check-success=e2e (3.11, 1.75, rust_vendor, ubuntu-latest)
+          - check-success=e2e (3.11, 1.75, lint_requirements, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, macos-latest)
           - check-success=e2e (3.12, 1.75, bootstrap, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, bootstrap_build_tags, macos-latest)
@@ -92,6 +93,8 @@ pull_request_rules:
           - check-success=e2e (3.12, 1.75, report_missing_dependency, ubuntu-latest)
           - check-success=e2e (3.12, 1.75, rust_vendor, macos-latest)
           - check-success=e2e (3.12, 1.75, rust_vendor, ubuntu-latest)
+          - check-success=e2e (3.12, 1.75, lint_requirements, macos-latest)
+          - check-success=e2e (3.12, 1.75, lint_requirements, ubuntu-latest)
           - "-draft"
 
           # At least 1 reviewer

e2e/test_lint_requirements.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# This script acts as an e2e test for lint_requirements command of fromager
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+pass=true
+
+# Test to demonstrate that command works as expected when input files are valid
+
+if ! fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/requirements.txt"; then
+    echo "the input files are not correctly formatted." 1>&2
+    pass=false
+fi;
+
+# Test to demonstrate failing of command due to missing input args
+
+if fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt"; then
+    echo "input arguments are missing for the command" 1>&2
+    pass=false
+fi;
+
+# Test to demonstrate that command reports error for invalid / bad input files
+
+if fromager lint-requirements "$SCRIPTDIR/validate_inputs/constraints.txt" "$SCRIPTDIR/validate_inputs/invalid-requirements.txt"; then
+    echo "input arguments are missing for the command" 1>&2
+    pass=false
+fi;
+
+$pass

e2e/validate_inputs/constraints.txt

@@ -0,0 +1,5 @@
+# This requirements.txt is used for testing lint_requirements
+# command of fromager and contains examples of packages
+
+gast==0.5.5
+torch==2.3.1

e2e/validate_inputs/invalid-requirements.txt

@@ -0,0 +1,5 @@
+# This requirements file is invalid on purpose to test the lint-requirements 
+# command of fromager. Do not attempt to fix this file
+
+foo==
+bar<=

e2e/validate_inputs/requirements.txt

@@ -0,0 +1,5 @@
+# This requirements.txt is used for testing lint_requirements
+# command of fromager and contains examples of packages
+stevedore==5.2.0
+kfp==2.11.0
+

src/fromager/commands/__init__.py

@@ -6,6 +6,7 @@
     download_sequence,
     graph,
     lint,
+    lint_requirements,
     list_overrides,
     migrate_config,
     server,
@@ -26,4 +27,5 @@
     download_sequence.download_sequence,
     server.wheel_server,
     migrate_config.migrate_config,
+    lint_requirements.lint_requirements,
 ]

src/fromager/commands/lint_requirements.py

@@ -0,0 +1,52 @@
+import fileinput
+import glob
+import itertools
+import logging
+import sys
+
+import click
+from packaging.requirements import InvalidRequirement, Requirement
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument("constraints_wildcard", type=click.Path(exists=False))
+@click.argument("requirements_wildcard", type=click.Path(exists=False))
+def lint_requirements(constraints_wildcard: str, requirements_wildcard: str) -> None:
+    """
+    Command to lint the constraints.txt and requirements.txt files
+    This command takes two wildcard paths for constraints.txt and requirements.txt respectively.
+    It checks the formatting of these files and reports issues if found.
+    """
+    # Exit if user does not provide wildcard paths
+    if not constraints_wildcard or not requirements_wildcard:
+        logger.error("path for requirements.txt or constraints.txt is missing")
+        sys.exit(1)
+
+    # Get all the files to check in a list
+    files_to_check = list(
+        itertools.chain.from_iterable(
+            glob.glob(f) for f in [constraints_wildcard, requirements_wildcard]
+        )
+    )
+
+    if len(files_to_check) == 0:
+        logger.error("no constraints.txt or requirements.txt found in given paths")
+        sys.exit(1)
+
+    flag = True
+
+    for line in fileinput.input(files_to_check):
+        string_to_check, _, _ = line.partition("#")
+        if not string_to_check.strip():
+            continue
+
+        try:
+            Requirement(string_to_check)
+        except InvalidRequirement as err:
+            logger.error(f"{fileinput.filename()}: {fileinput.filelineno()}: {err}")
+            flag = False
+
+    if not flag:
+        sys.exit(1)