From fb5fad63f4595c66121bbb308ffe507ec6cba279 Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Wed, 7 Jun 2023 17:02:54 +0200 Subject: [PATCH] Fixing release notes for new major releases Parent issue: https://github.com/sequentech/meta/issues/111 Fixing major release note procedure ensuring Release Flow support works. Previously, when creating a new major release lots of issues were duplicated from previous minor releases. --- comprehensive_release_notes.py | 42 ++++++++++++- release.py | 107 ++++++++++++++++++++++++--------- release_notes.py | 47 ++++++++++++--- 3 files changed, 156 insertions(+), 40 deletions(-) diff --git a/comprehensive_release_notes.py b/comprehensive_release_notes.py index d884b1d..4b03064 100644 --- a/comprehensive_release_notes.py +++ b/comprehensive_release_notes.py @@ -41,13 +41,16 @@ "sequentech/release-tool", ] -def get_comprehensive_release_notes(args, token, repos, prev_release, new_release, config): +def get_comprehensive_release_notes( + args, token, repos, prev_major_release, prev_release, new_release, config +): """ Generate comprehensive release notes for a list of repositories. Args: token (str): GitHub access token. repos (list): A list of repository paths, e.g., ["org/repo1", "org/repo2"]. + prev_major_release (str|None): The previous major release version (e.g. "1.0.0") or None if prev_release and new_release share their major version. prev_release (str): The previous release version (e.g. "1.1.0"). new_release (str): The new release version (e.g. "1.2.0"). config (dict): the configuration for generating release notes. @@ -60,7 +63,22 @@ def get_comprehensive_release_notes(args, token, repos, prev_release, new_releas for repo_path in repos: verbose_print(args, f"Generating release notes for repo {repo_path}..") repo = gh.get_repo(repo_path) - repo_notes = get_release_notes(gh, repo, prev_release, new_release, config) + + hidden_links = [] + # if we are going to do a new major release for example + # new_release="8.0.0", we need to obtain a list of all the changes made + # in the previous major release cycle (from 7.0.0 to + # previous_release="7.4.0") and mark them as hidden. + if prev_major_release: + verbose_print(args, f"Generating release notes for hidden links:") + (_, hidden_links) = get_release_notes( + gh, repo, prev_major_release, prev_release, config, hidden_links=[] + ) + + verbose_print(args, f"Generating release notes:") + (repo_notes, _) = get_release_notes( + gh, repo, prev_release, new_release, config, hidden_links=hidden_links + ) verbose_print(args, f"..generated") for category, notes in repo_notes.items(): release_notes[category].extend(notes) @@ -153,8 +171,26 @@ def main(): verbose_print(args, f"Previous Release Head: {prev_release_head}") verbose_print(args, f"New Release Head: {new_release_head}") + if prev_major != new_major: + # if we are going to do a new major release for example + # new_release="8.0.0", we need to obtain a list of all the changes made + # in the previous major release cycle (from 7.0.0 to + # previous_release="7.4.0") and mark them as hidden. + prev_major_release_head = get_release_head(prev_major, 0, "0") + else: + prev_major_release_head = None + verbose_print( + args, + f"Previous Major Release Head: {prev_major_release_head}" + ) + release_notes = get_comprehensive_release_notes( - args, github_token, REPOSITORIES, prev_release_head, new_release_head, + args, + github_token, + REPOSITORIES, + prev_major_release_head, + prev_release_head, + new_release_head, config ) diff --git a/release.py b/release.py index f4cc454..c6e2937 100755 --- a/release.py +++ b/release.py @@ -16,6 +16,7 @@ # along with release-tool. If not, see . import argparse +import yaml import requests import tempfile from datetime import datetime @@ -23,6 +24,14 @@ import os import re from jinja2 import Environment, FileSystemLoader, select_autoescape +from github import Github +from collections import defaultdict +from release_notes import ( + get_sem_release, + get_release_head, + get_release_notes, + create_release_notes_md, +) def read_text_file(file_path): textfile = open(file_path, "r") @@ -618,35 +627,69 @@ def do_create_release( previous_tag_name, prerelease ): - with tempfile.NamedTemporaryFile() as temp_release_file: - generated_release_title = '' - if generate_release_notes: - dir_name = os.path.basename(dir_path) - data = { - 'tag_name': version, - } - if previous_tag_name is not None: - data['previous_tag_name'] = previous_tag_name - req = requests.post( - f'https://api.github.com/repos/sequentech/{dir_name}/releases/generate-notes', - headers={ - "Accept": "application/vnd.github.v3+json" - }, - json=data, - auth=( - os.getenv('GITHUB_USER'), - os.getenv('GITHUB_TOKEN'), - ) + if not generate_release_notes: + release_notes_md = "" + else: + github_token = os.getenv("GITHUB_TOKEN") + + gh = Github(github_token) + release_notes = defaultdict(list) + project_name = os.path.basename(dir_path) + repo_path = f"sequentech/{project_name}" + print(f"Generating release notes for repo {repo_path}..") + repo = gh.get_repo(repo_path) + + with open(".github/release.yml") as release_template_yaml: + config = yaml.safe_load(release_template_yaml) + + prev_major, prev_minor, prev_patch = get_sem_release(previous_tag_name) + new_major, new_minor, new_patch = get_sem_release(version) + + prev_release_head = get_release_head(prev_major, prev_minor, prev_patch) + if new_patch or prev_major == new_major: + new_release_head = get_release_head(new_major, new_minor, new_patch) + else: + new_release_head = repo.default_branch + + print(f"Previous Release Head: {prev_release_head}") + print(f"New Release Head: {new_release_head}") + if prev_major != new_major: + # if we are going to do a new major release for example + # new_release="8.0.0", we need to obtain a list of all the changes made + # in the previous major release cycle (from 7.0.0 to + # previous_release="7.4.0") and mark them as hidden. + prev_major_release_head = get_release_head(prev_major, 0, "0") + else: + prev_major_release_head = None + print(f"Previous Major Release Head: {prev_major_release_head}") + + hidden_links = [] + # if we are going to do a new major release for example + # new_release="8.0.0", we need to obtain a list of all the changes made + # in the previous major release cycle (from 7.0.0 to + # previous_release="7.4.0") and mark them as hidden. + if prev_major_release_head: + print(f"Generating release notes for hidden links:") + (_, hidden_links) = get_release_notes( + gh, repo, prev_major_release_head, prev_release_head, config, + hidden_links=[] ) - if req.status_code != 200: - print(f"Error generating release notes, status ${req.status_code}") - exit(1) - - generated_release_notes = req.json()['body'] - temp_release_file.write(generated_release_notes.encode('utf-8')) - temp_release_file.flush() - generated_release_title = req.json()['name'] - print(f"- github-generated release notes:\n\n{generated_release_notes}\n\n") + + print(f"Generating release notes:") + (repo_notes, _) = get_release_notes( + gh, repo, prev_release_head, new_release_head, config, + hidden_links=hidden_links + ) + print(f"..generated") + for category, notes in repo_notes.items(): + release_notes[category].extend(notes) + release_notes_md = create_release_notes_md(release_notes, new_release_head) + print(f"Generated Release Notes markdown: {release_notes_md}") + generated_release_title = f"{new_release_head} release" + + with tempfile.NamedTemporaryFile() as temp_release_file: + temp_release_file.write(release_notes_md.encode('utf-8')) + temp_release_file.flush() print("checking if release exists to overwrite it..") ret_code = call_process( @@ -846,6 +889,14 @@ def main(): nargs="+", help="Set the dependabot alerts only for the given repository branches" ) + parser.add_argument( + '--dry-run', + action='store_true', + help=( + 'Output the release notes but do not create any tag, release or ' + 'new branch.' + ) + ) args = parser.parse_args() change_version = args.change_version version = args.version diff --git a/release_notes.py b/release_notes.py index e5ec806..f5cd182 100644 --- a/release_notes.py +++ b/release_notes.py @@ -10,6 +10,14 @@ from datetime import datetime from github import Github +# Text to detect in PR descriptions which is followed by the link to the parent +# issue +PARENT_ISSUE_TEXT = 'Parent issue: ' + +# If a commit is older than this, we will ignore it in the release notes because +# we were not following the same github procedures at that time +CUTOFF_DATE = 'Sun, 01 Jan 2023 00:00:00 GMT' + def get_label_category(labels, categories): """ Get the category that matches the given labels. @@ -87,7 +95,15 @@ def get_github_issue_from_link(link_text, github): return issue -def get_release_notes(github, repo, previous_release_head, new_release_head, config, args=type('', (), {'silent': False})()): +def get_release_notes( + github, + repo, + previous_release_head, + new_release_head, + config, + hidden_links=[], + args=type('', (), {'silent': False})() + ): """ Retrieve release notes from a GitHub repository based on the given configuration. @@ -96,13 +112,20 @@ def get_release_notes(github, repo, previous_release_head, new_release_head, con :param previous_release_head: str, the previous release's head commit. :param new_release_head: str, the new release's head commit. :param config: dict, the configuration for generating release notes. - :return: dict, the release notes categorized by their labels. + :return: tuple with ( + dict: the release notes categorized by their labels, + list: list of links to the PRs included + ) """ compare_branches = repo.compare(previous_release_head, new_release_head) release_notes = {} parent_issues = [] links = [] + cutoff_date = datetime.strptime( + CUTOFF_DATE, + '%a, %d %b %Y %H:%M:%S %Z' + ) for commit in compare_branches.commits: pr = get_commit_pull(commit) @@ -117,12 +140,16 @@ def get_release_notes(github, repo, previous_release_head, new_release_head, con continue title = pr.title.strip() - parent_issue_text = "Parent issue: " + + if pr.closed_at < cutoff_date: + verbose_print(args, f"[before cut-off date]ignoring PR: {title}: {pr.html_url}\n") + continue + parent_issue = None if isinstance(pr.body, str): for line in pr.body.split("\n"): - if line.startswith(parent_issue_text): - parent_issue = line[len(parent_issue_text):] + if line.startswith(PARENT_ISSUE_TEXT): + parent_issue = line[len(PARENT_ISSUE_TEXT):] break if parent_issue: @@ -135,8 +162,8 @@ def get_release_notes(github, repo, previous_release_head, new_release_head, con title = issue.title.strip() else: link = pr.html_url - - if link in links: + + if link in links or link in hidden_links: continue else: links.append(link) @@ -149,7 +176,7 @@ def get_release_notes(github, repo, previous_release_head, new_release_head, con release_notes_yaml = yaml.dump(release_notes, default_flow_style=False) verbose_print(args, f"release notes:\n{release_notes_yaml}") - return release_notes + return (release_notes, links) def create_release_notes_md(release_notes, new_release): """ @@ -310,7 +337,9 @@ def main(): verbose_print(args, f"Previous Release Head: {prev_release_head}") verbose_print(args, f"New Release Head: {new_release_head}") - release_notes = get_release_notes(gh, repo, prev_release_head, new_release_head, config, args) + (release_notes, _) = get_release_notes( + gh, repo, prev_release_head, new_release_head, config, args + ) if not new_patch: latest_release = repo.get_releases()[0]