Merge pull request #14 from chrisburr/add-release-notes

Add full release notes

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,13 @@
+
+
+BEGINRELEASENOTES
+Thank you for writing the text to appear in the release notes. It will show up
+exactly as it appears between the two bold lines
+
+Please follow the template:
+*Subsystem
+NEW/CHANGE/FIX: explanation
+
+For examples look into release.notes
+
+ENDRELEASENOTES

.github/workflows/release.yaml

@@ -26,7 +26,15 @@ jobs:
       - name: Create release
         run: |
           eval "$(${HOME}/miniforge/bin/conda shell.bash hook)"
+
+          wget https://raw.githubusercontent.com/chrisburr/DIRAC/releases-notes-token-arg/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py
+          python dirac-docs-get-release-notes.py \
+            -r DIRACGrid/DIRACOS2 \
+            --branches main \
+            --token="${{ secrets.GITHUB_TOKEN }}" | tail -n +2 | sed '/^$/d' > release_notes.txt
+
           scripts/make_release.py \
             --token="${{ secrets.GITHUB_TOKEN }}" \
             --run-id="${{ github.event.inputs.run_id }}" \
-            --version="${{ github.event.inputs.version }}"
+            --version="${{ github.event.inputs.version }}" \
+            --release-notes-fn="$PWD/release_notes.txt"

scripts/make_release.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python
 import argparse
 import base64
+import difflib
 from io import BytesIO
 import re
 import zipfile
 
-from packaging.version import Version
+from packaging.version import Version, InvalidVersion
 import requests
 from uritemplate import expand as uri_expand
 
@@ -14,7 +15,12 @@
 END_HEADER_MAGIC = b"@@END_HEADER@@"
 
 
-def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"):
+def main(
+    run_id=None,
+    requested_version=None,
+    workflow_fn="build-and-test.yml",
+    release_notes_fn=None,
+):
     """Make a new release of DIRACOS2 based on a GitHub Actions CI run
 
     Releases are made by:
@@ -34,6 +40,7 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"):
     :param str requested_version: A version to embed in the DIRACOS2 installer, overriding
         the pre-existing value from when it was built
     :param str workflow_fn: The name of the GitHub Actions workflow configuration file
+    :param str release_notes_fn: Path to a file containing release notes
     """
     if run_id is None:
         # Find the run pipeline for the main branch
@@ -58,13 +65,22 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"):
     print(f"Releasing {this_version} next version will be {next_version}")
     # There should be once instance of the version string in the header and
     # the rest should be "DIRACOS $VER". Check this is the case.
-    assert header.count(installer_metadata["VER"]) == header.count(f"DIRACOS {installer_metadata['VER']}") + 1
+    assert (
+        header.count(installer_metadata["VER"])
+        == header.count(f"DIRACOS {installer_metadata['VER']}") + 1
+    )
     # Update the version in the installer to be the requested one
     header = header.replace(installer_metadata["VER"], this_version)
     installer = header.encode() + END_HEADER_MAGIC + installer_data
 
+    # Load the release notes
+    release_notes = ""
+    if release_notes_fn:
+        with open(release_notes_fn) as fp:
+            release_notes = fp.read()
+
     # Create the GitHub release
-    make_release(installer, environment_yaml, this_version, commit_hash)
+    make_release(installer, environment_yaml, this_version, commit_hash, release_notes)
 
     # Update the construct.yaml on main
     bump_version_in_main(next_version)
@@ -81,13 +97,11 @@ def get_artifacts_zip(artifacts, artifact_name):
     r = requests.get(
         artifacts[artifact_name]["archive_download_url"],
         headers={"Authorization": f"token {token}"},
-        stream=True
+        stream=True,
     )
     r.raise_for_status()
-    decoding_classes = {
-        "application/zip": zipfile.ZipFile
-    }
-    return decoding_classes[r.headers['Content-Type']](BytesIO(r.content))
+    decoding_classes = {"application/zip": zipfile.ZipFile}
+    return decoding_classes[r.headers["Content-Type"]](BytesIO(r.content))
 
 
 def get_version(version, installer_metadata):
@@ -116,7 +130,7 @@ def get_version(version, installer_metadata):
         # Bump the least significant digit
         next_version = [
             ".".join(map(str, version.release[:-1] + (version.release[-1] + 1,))),
-            "a1"
+            "a1",
         ]
 
     return str(version), "".join(next_version)
@@ -136,13 +150,19 @@ def get_installer_artifacts(run_id):
     r.raise_for_status()
     run_info = r.json()
     if run_info["conclusion"] != "success":
-        raise RuntimeError(f"Run {run_id} has not succeeded, its status is {run_info['conclusion']}")
+        raise RuntimeError(
+            f"Run {run_id} has not succeeded, its status is {run_info['conclusion']}"
+        )
 
     r = requests.get(f"{api_root}/actions/runs/{run_id}/artifacts", headers=headers)
     r.raise_for_status()
     artifacts = {x["name"]: x for x in r.json()["artifacts"]}
 
-    environment_yaml = get_artifacts_zip(artifacts, "environment-yaml").read("environment.yaml").decode()
+    environment_yaml = (
+        get_artifacts_zip(artifacts, "environment-yaml")
+        .read("environment.yaml")
+        .decode()
+    )
 
     installer_zip = get_artifacts_zip(artifacts, "installer")
     if len(installer_zip.filelist) != 1:
@@ -154,7 +174,72 @@ def get_installer_artifacts(run_id):
     return run_info["head_sha"], environment_yaml, installer
 
 
-def make_release(installer, environment_yaml, version, commit_hash):
+def changes_since_last_version(version, environment_yaml):
+    """Generate diffs of the packages installed since the previous version
+
+    :param str version: The version of the new release
+    :param str environment_yaml: The `str` corresponding to the ``environment.yaml``
+        generated by the DIRACOS2 installer
+    :returns: A tuple of the:
+        * The version number of the previous release
+        * A diff of changes to the environment.yaml specification with build strings ignored
+        * A diff of changes to the environment.yaml specification
+    """
+    # Find the previous release
+    r = requests.get(f"{api_root}/releases", params={"per_page": 100}, headers=headers)
+    r.raise_for_status()
+    releases = {}
+    for release in r.json():
+        for asset in release["assets"]:
+            if asset["name"] == "DIRACOS-environment.yaml":
+                break
+        else:
+            raise NotImplementedError(
+                f"Failed to find DIRACOS-environment.yaml for {release['tag_name']}"
+            )
+        try:
+            tag_version = Version(release["tag_name"])
+        except InvalidVersion:
+            print("Skipping invalid tag", release["tag_name"])
+        else:
+            releases[tag_version] = asset["browser_download_url"]
+    version = Version(version)
+    sorted_releases = sorted(list(releases) + [version])
+    previous_version = sorted_releases[sorted_releases.index(version) - 1]
+
+    # Download the previous release's environment.yaml
+    previous_yaml_url = releases[previous_version]
+    r = requests.get(previous_yaml_url)
+    r.raise_for_status()
+    previous_environment_yaml = r.text
+
+    # Generate the full diff
+    diff_full = "\n".join(
+        difflib.unified_diff(
+            previous_environment_yaml.split("\n"),
+            environment_yaml.split("\n"),
+            n=0,
+            lineterm="",
+        )
+    )
+
+    # Generate the diff ignoring build numbers
+    pattern = re.compile(r"(^\s+\- [^=]+=[^=]+)=[^=]+$", re.MULTILINE)
+    previous_environment_yaml = pattern.sub(r"\1", previous_environment_yaml)
+    environment_yaml = pattern.sub(r"\1", environment_yaml)
+    diff_without_builds = "\n".join(
+        difflib.unified_diff(
+            previous_environment_yaml.split("\n"),
+            environment_yaml.split("\n"),
+            n=0,
+            lineterm="",
+        )
+    )
+
+    return str(previous_version), diff_without_builds, diff_full
+
+
+def make_release(installer, environment_yaml, version, commit_hash, release_notes):
     """Create a new GitHub release using the given data
 
     This function always makes a pre-release first to ensure the "latest" release never corresponds
@@ -167,28 +252,54 @@ def make_release(installer, environment_yaml, version, commit_hash):
         generated by the DIRACOS2 installer
     :param str version: The version of the new release
     :param str commit_hash: DIRACOS2 git revision used for the release
+    :param str release_notes: Release notes to include in the release description
     """
-    release_notes = "\n".join([
-        f"# DIRACOS {version}",
-        "",
-        "## Changes",
-        "* TODO",
-        "",
-        "## Package list",
-        "```yaml",
-        environment_yaml,
-        "```",
-    ])
+    previous_version, diff_without_builds, diff_full = changes_since_last_version(
+        version, environment_yaml
+    )
+
+    release_notes = "\n".join(
+        [
+            f"# DIRACOS {version}",
+            "",
+            "## Release notes",
+            release_notes,
+            "",
+            "## Package list",
+            "<details>",
+            "  <summary>Click to expand!</summary>",
+            "```yaml",
+            environment_yaml,
+            "```",
+            "</details>",
+            "",
+            f"## Changes with respect to {previous_version} ignoring build strings",
+            "<details>",
+            "  <summary>Click to expand!</summary>",
+            "```diff",
+            diff_without_builds,
+            "```",
+            "</details>",
+            "",
+            f"## Full changes with respect to {previous_version}",
+            "<details>",
+            "  <summary>Click to expand!</summary>",
+            "```diff",
+            diff_full,
+            "```",
+            "</details>",
+        ]
+    )
 
     # Create a draft release
     r = requests.post(
         f"{api_root}/releases",
         json={
-        "tag_name": version,
-        "target_commitish": commit_hash,
-        "body": release_notes,
-        "draft": True,
-        "prerelease": Version(version).is_prerelease,
+            "tag_name": version,
+            "target_commitish": commit_hash,
+            "body": release_notes,
+            "draft": True,
+            "prerelease": Version(version).is_prerelease,
         },
         headers=headers,
     )
@@ -270,7 +381,9 @@ def bump_version_in_main(new_version):
     else:
         raise NotImplementedError("Failed to find the version from construct.yaml")
 
-    new_data, num_subs = CONSTRUCT_VERSION_PATTERN.subn(f"version: {new_version}\n", data)
+    new_data, num_subs = CONSTRUCT_VERSION_PATTERN.subn(
+        f"version: {new_version}\n", data
+    )
     if num_subs != 1:
         raise RuntimeError(num_subs)
 
@@ -284,7 +397,9 @@ def bump_version_in_main(new_version):
         headers=headers,
     )
     r.raise_for_status()
-    print(f"Pushed commit to bump version to {new_version} as {r.json()['commit']['html_url']}")
+    print(
+        f"Pushed commit to bump version to {new_version} as {r.json()['commit']['html_url']}"
+    )
 
 
 # Crude unit tests for get_version
@@ -301,6 +416,7 @@ def bump_version_in_main(new_version):
     parser.add_argument("--repo", default="DIRACOS2")
     parser.add_argument("--run-id")
     parser.add_argument("--version")
+    parser.add_argument("--release-notes-fn")
     args = parser.parse_args()
 
     token = args.token
@@ -315,5 +431,6 @@ def bump_version_in_main(new_version):
 
     main(
         run_id=int(args.run_id) if args.run_id else None,
-        requested_version=str(Version(args.version)) if args.version else None
+        requested_version=str(Version(args.version)) if args.version else None,
+        release_notes_fn=args.release_notes_fn,
     )