Update CI files

.ci/ansible/Containerfile.j2

@@ -9,20 +9,27 @@ ADD ./{{ item.name }} ./{{ item.name }}
 # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install
+# This MUST be the ONLY call to pip install in inside the container.
+RUN pip3 install --upgrade pip setuptools wheel && \
+  rm -rf /root/.cache/pip && \
+  pip3 install pipdeptree
 {%- if s3_test | default(false) -%}
 {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
 {%- endif -%}
 {%- for item in plugins -%}
 {{ " " }}{{ item.source }}
+{%- if item.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+{%- endif -%}
 {%- if item.lowerbounds | default(false) -%}
 {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
 {%- endif -%}
 {%- if item.ci_requirements | default(false) -%}
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
+  rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}
 {% for key, value in pulp_env.items() %}

.ci/scripts/calc_constraints.py

@@ -0,0 +1,118 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import argparse
+import fileinput
+import urllib.request
+import sys
+from packaging.requirements import Requirement
+from packaging.version import Version
+import yaml
+
+
+CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
+
+
+def fetch_pulpcore_upper_bound(requirement):
+    with urllib.request.urlopen(CORE_TEMPLATE_URL) as f:
+        template = yaml.safe_load(f.read())
+    supported_versions = template["supported_release_branches"]
+    supported_versions.append(template["latest_release_branch"])
+    applicable_versions = sorted(
+        requirement.specifier.filter((Version(v) for v in supported_versions))
+    )
+    if len(applicable_versions) == 0:
+        raise Exception("No supported pulpcore version in required range.")
+    return f"{requirement.name}~={applicable_versions[-1]}"
+
+
+def split_comment(line):
+    split_line = line.split("#", maxsplit=1)
+    try:
+        comment = "  # " + split_line[1].strip()
+    except IndexError:
+        comment = ""
+    return split_line[0].strip(), comment
+
+
+def to_upper_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        if requirement.name == "pulpcore":
+            # An exception to allow for pulpcore deprecation policy.
+            return fetch_pulpcore_upper_bound(requirement)
+        for spec in requirement.specifier:
+            if spec.operator == "~=":
+                return f"# NO BETTER CONSTRAINT: {req}"
+            if spec.operator == "<=":
+                operator = "=="
+                max_version = spec.version
+                return f"{requirement.name}{operator}{max_version}"
+            if spec.operator == "<":
+                operator = "~="
+                version = Version(spec.version)
+                if version.micro != 0:
+                    max_version = f"{version.major}.{version.minor}.{version.micro-1}"
+                elif version.minor != 0:
+                    max_version = f"{version.major}.{version.minor-1}"
+                elif version.major != 0:
+                    max_version = f"{version.major-1}.0"
+                else:
+                    return f"# NO BETTER CONSTRAINT: {req}"
+                return f"{requirement.name}{operator}{max_version}"
+        return f"# NO UPPER BOUND: {req}"
+
+
+def to_lower_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        for spec in requirement.specifier:
+            if spec.operator == ">=":
+                if requirement.name == "pulpcore":
+                    # Currently an exception to allow for pulpcore bugfix releases.
+                    # TODO Semver libraries should be allowed too.
+                    operator = "~="
+                else:
+                    operator = "=="
+                min_version = spec.version
+                return f"{requirement.name}{operator}{min_version}"
+        return f"# NO LOWER BOUND: {req}"
+
+
+def main():
+    """Calculate constraints for the lower bound of dependencies where possible."""
+    parser = argparse.ArgumentParser(
+        prog=sys.argv[0],
+        description="Calculate constraints for the lower or upper bound of dependencies where "
+        "possible.",
+    )
+    parser.add_argument("-u", "--upper", action="store_true")
+    parser.add_argument("filename", nargs="*")
+    args = parser.parse_args()
+
+    with fileinput.input(files=args.filename) as req_file:
+        for line in req_file:
+            if line.strip().startswith("#"):
+                # Shortcut comment only lines
+                print(line.strip())
+            else:
+                req, comment = split_comment(line)
+                if args.upper:
+                    new_req = to_upper_bound(req)
+                else:
+                    new_req = to_lower_bound(req)
+                print(new_req + comment)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/collect_changes.py

@@ -103,7 +103,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/pr_labels.py

@@ -0,0 +1,57 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    CHANGELOG_EXTS = [
+        f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+    ]
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/validate_commit_message.py

@@ -9,21 +9,16 @@
 import sys
 from pathlib import Path
 import subprocess
-
-
 import os
 import warnings
 from github import Github
 
-
-NO_ISSUE = "[noissue]"
 CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+KEYWORDS = ["fixes", "closes"]
+
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
-
-KEYWORDS = ["fixes", "closes"]
-
 g = Github(os.environ.get("GITHUB_TOKEN"))
 repo = g.get_repo("pulp/pulp_rpm")
 
@@ -64,15 +59,5 @@ def __check_changelog(issue):
     for issue in pattern.findall(message):
         __check_status(issue)
         __check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    elif "Merge" in message and "cherry picked from commit" in message:
-        pass
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-349-gba81617
+2021.08.26-382-gc88324b

.github/workflows/ci.yml

@@ -46,6 +46,9 @@ jobs:
         run: |
           python .ci/scripts/check_requirements.py
 
+  docs:
+    uses: "./.github/workflows/docs.yml"
+
   lint:
     uses: "./.github/workflows/lint.yml"
 
@@ -84,6 +87,7 @@ jobs:
       - "check-commits"
       - "lint"
       - "test"
+      - "docs"
     if: "always()"
     steps:
       - name: "Collect needed jobs results"