From 282c2462c4d8c3102dba0264cb98235e8e7cdaeb Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 24 Nov 2020 15:01:34 +0100 Subject: [PATCH 1/5] Add docstrings to make_release.py (#8) --- scripts/make_release.py | 171 +++++++++++++++++++++++++++++++--------- 1 file changed, 134 insertions(+), 37 deletions(-) diff --git a/scripts/make_release.py b/scripts/make_release.py index f649b16..d999ec0 100755 --- a/scripts/make_release.py +++ b/scripts/make_release.py @@ -2,7 +2,6 @@ import argparse import base64 from io import BytesIO -import os import re import zipfile @@ -15,43 +14,31 @@ END_HEADER_MAGIC = b"@@END_HEADER@@" -def get_artifacts_zip(artifacts, artifact_name): - r = requests.get( - artifacts[artifact_name]["archive_download_url"], - headers={"Authorization": f"token {token}"}, - stream=True - ) - r.raise_for_status() - decoding_classes = { - "application/zip": zipfile.ZipFile - } - return decoding_classes[r.headers['Content-Type']](BytesIO(r.content)) - - -def get_version(version, installer_metadata): - if version is None: - # If no version is given, convert the build version number to a release version - version = Version(installer_metadata["VER"]) - version = ".".join(map(str, version.release)) - version = Version(version) - - if version.is_prerelease: - # Bump the pre-release digit - next_version = [ - ".".join(map(str, version.release)), - "".join(map(str, version.pre[:-1] + (version.pre[-1] + 1,))), - ] - else: - # Bump the least significant digit - next_version = [ - ".".join(map(str, version.release[:-1] + (version.release[-1] + 1,))), - "a1" - ] - - return str(version), "".join(next_version) - - def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): + """Make a new release of DIRACOS2 based on a GitHub Actions CI run + + Releases are made by: + 1. Download the artefact from a GitHub Actions CI run + 2. Editing it in place to change the version number + 3. Creating a pre-release in GitHub + 4. Uploading release artefacts to the new pre-release + 5. If the version number is not a pre-release: converting the pre-release to a full release + causing the "latest" release to update + 6. Editing the current "main" branch to be one version higher than the current latest release + + Authentication is performed using the GitHub API token which is automatically made available + to all GitHub Actions jobs. + + Parameters + ---------- + run_id : str or None + The GitHub Actions run ID or, if missing, defaults to the most recent + requested_version : str or None + A version to embed in the DIRACOS2 installer, overriding the pre-existing value from when + it was built + workflow_fn : str + The name of the GitHub Actions workflow configuration file + """ if run_id is None: # Find the run pipeline for the main branch r = requests.get( @@ -87,7 +74,91 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): bump_version_in_main(next_version) +def get_artifacts_zip(artifacts, artifact_name): + """Download an artifact from a GitHub actions CI run + + Parameters + ---------- + artifacts : dict + Mapping of artifacts names to data from + https://developer.github.com/v3/actions/artifacts/#list-workflow-run-artifacts + artifact_name : str + The key in the dictionary to download + + Returns + ------- + An in-memory zipfile.ZipFile object + """ + r = requests.get( + artifacts[artifact_name]["archive_download_url"], + headers={"Authorization": f"token {token}"}, + stream=True + ) + r.raise_for_status() + decoding_classes = { + "application/zip": zipfile.ZipFile + } + return decoding_classes[r.headers['Content-Type']](BytesIO(r.content)) + + +def get_version(version, installer_metadata): + """Extract the version from the metadata written by conda-constructor + + This function is fragile and based on the implementation of conda-constructor in + https://github.com/conda/constructor/blob/master/constructor/header.sh + + Parameters + ---------- + version : str or None + Override the version data contained in the DIRACOS2 installer with this version number + installer_metadata : dict + The metadata extracted from the header of the DIRACOS2 installer + + Returns + ------- + A tuple of ``str`` containing (current_version, next_version) + """ + if version is None: + # If no version is given, convert the build version number to a release version + version = Version(installer_metadata["VER"]) + version = ".".join(map(str, version.release)) + version = Version(version) + + if version.is_prerelease: + # Bump the pre-release digit + next_version = [ + ".".join(map(str, version.release)), + "".join(map(str, version.pre[:-1] + (version.pre[-1] + 1,))), + ] + else: + # Bump the least significant digit + next_version = [ + ".".join(map(str, version.release[:-1] + (version.release[-1] + 1,))), + "a1" + ] + + return str(version), "".join(next_version) + + def get_installer_artifacts(run_id): + """Download the artefacts from a GitHub Actions run + + Parameters + ---------- + run_id: int + The GitHub Actions run ID to download artifacts from + + Returns + ------- + A tuple of the: + * Git revision used for the given Run ID + * The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer + * The `bytes` corresponding to the DIRACOS2 installer itself + + Raises + ------ + A ``RuntimeError`` if the GitHub Actions run was not successful + """ r = requests.get(f"{api_root}/actions/runs/{run_id}", headers=headers) r.raise_for_status() run_info = r.json() @@ -111,6 +182,25 @@ def get_installer_artifacts(run_id): def make_release(installer, environment_yaml, version, commit_hash): + """Create a new GitHub release using the given data + + This function always makes a pre-release first to ensure the "latest" release never corresponds + to one without artefacts uploaded. If the new version number is not a pre-release, as + determined by PEP-440, it is promoted to at full release after he uploads have completed + successfully. + + Parameters + ---------- + installer : bytes + The `bytes` corresponding to the DIRACOS2 installer itself + environment_yaml : str + The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer + version : str + The version of the new release + + commit_hash : str + DIRACOS2 git revision used for the release + """ release_notes = "\n".join([ f"# DIRACOS {version}", "", @@ -197,6 +287,13 @@ def make_release(installer, environment_yaml, version, commit_hash): def bump_version_in_main(new_version): + """Edit the construct.yaml file on main to correspond to the next DIRACOS2 version + + Parameters + ---------- + new_version : str + The next version number of DIRACOS2 + """ r = requests.get(f"{api_root}/contents/construct.yaml", headers=headers) r.raise_for_status() file_info = r.json() From 2c8b3f131a7305e791a7e8373ebad6b92d0b22d9 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 8 Dec 2020 11:35:47 +0100 Subject: [PATCH 2/5] Apply suggestions from code review Co-authored-by: Andre Sailer --- scripts/make_release.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/make_release.py b/scripts/make_release.py index d999ec0..00070cf 100755 --- a/scripts/make_release.py +++ b/scripts/make_release.py @@ -18,10 +18,10 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): """Make a new release of DIRACOS2 based on a GitHub Actions CI run Releases are made by: - 1. Download the artefact from a GitHub Actions CI run + 1. Download the artifact from a GitHub Actions CI run 2. Editing it in place to change the version number 3. Creating a pre-release in GitHub - 4. Uploading release artefacts to the new pre-release + 4. Uploading release artifacts to the new pre-release 5. If the version number is not a pre-release: converting the pre-release to a full release causing the "latest" release to update 6. Editing the current "main" branch to be one version higher than the current latest release @@ -141,7 +141,7 @@ def get_version(version, installer_metadata): def get_installer_artifacts(run_id): - """Download the artefacts from a GitHub Actions run + """Download the artifacts from a GitHub Actions run Parameters ---------- @@ -185,8 +185,8 @@ def make_release(installer, environment_yaml, version, commit_hash): """Create a new GitHub release using the given data This function always makes a pre-release first to ensure the "latest" release never corresponds - to one without artefacts uploaded. If the new version number is not a pre-release, as - determined by PEP-440, it is promoted to at full release after he uploads have completed + to one without artifacts uploaded. If the new version number is not a pre-release, as + determined by PEP-440, it is promoted to at full release after the uploads have completed successfully. Parameters From a56f03d5202a91642264ceed67c1b3f046aa641f Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 8 Dec 2020 11:40:13 +0100 Subject: [PATCH 3/5] Fix list formatting --- scripts/make_release.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/make_release.py b/scripts/make_release.py index 00070cf..6939802 100755 --- a/scripts/make_release.py +++ b/scripts/make_release.py @@ -18,6 +18,7 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): """Make a new release of DIRACOS2 based on a GitHub Actions CI run Releases are made by: + 1. Download the artifact from a GitHub Actions CI run 2. Editing it in place to change the version number 3. Creating a pre-release in GitHub @@ -151,9 +152,10 @@ def get_installer_artifacts(run_id): Returns ------- A tuple of the: - * Git revision used for the given Run ID - * The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer - * The `bytes` corresponding to the DIRACOS2 installer itself + + * Git revision used for the given Run ID + * The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer + * The `bytes` corresponding to the DIRACOS2 installer itself Raises ------ From a420c2ec1431351d757a7b764677b05ebd99b28a Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 8 Dec 2020 15:36:29 +0100 Subject: [PATCH 4/5] Use sphinx style docstrings --- scripts/make_release.py | 82 +++++++++++------------------------------ 1 file changed, 22 insertions(+), 60 deletions(-) diff --git a/scripts/make_release.py b/scripts/make_release.py index 6939802..7da0a6e 100755 --- a/scripts/make_release.py +++ b/scripts/make_release.py @@ -30,15 +30,10 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): Authentication is performed using the GitHub API token which is automatically made available to all GitHub Actions jobs. - Parameters - ---------- - run_id : str or None - The GitHub Actions run ID or, if missing, defaults to the most recent - requested_version : str or None - A version to embed in the DIRACOS2 installer, overriding the pre-existing value from when - it was built - workflow_fn : str - The name of the GitHub Actions workflow configuration file + :param str run_id: The GitHub Actions run ID or, if missing, defaults to the most recent + :param str requested_version: A version to embed in the DIRACOS2 installer, overriding + the pre-existing value from when it was built + :param str workflow_fn: The name of the GitHub Actions workflow configuration file """ if run_id is None: # Find the run pipeline for the main branch @@ -78,17 +73,10 @@ def main(run_id=None, requested_version=None, workflow_fn="build-and-test.yml"): def get_artifacts_zip(artifacts, artifact_name): """Download an artifact from a GitHub actions CI run - Parameters - ---------- - artifacts : dict - Mapping of artifacts names to data from + :param dict artifacts: Mapping of artifacts names to data from https://developer.github.com/v3/actions/artifacts/#list-workflow-run-artifacts - artifact_name : str - The key in the dictionary to download - - Returns - ------- - An in-memory zipfile.ZipFile object + :param str artifact_name: The key in the dictionary to download + :returns: An in-memory zipfile.ZipFile object """ r = requests.get( artifacts[artifact_name]["archive_download_url"], @@ -108,16 +96,9 @@ def get_version(version, installer_metadata): This function is fragile and based on the implementation of conda-constructor in https://github.com/conda/constructor/blob/master/constructor/header.sh - Parameters - ---------- - version : str or None - Override the version data contained in the DIRACOS2 installer with this version number - installer_metadata : dict - The metadata extracted from the header of the DIRACOS2 installer - - Returns - ------- - A tuple of ``str`` containing (current_version, next_version) + :param str version: Override the version data contained in the DIRACOS2 installer with this version number + :param dict installer_metadata: The metadata extracted from the header of the DIRACOS2 installer + :returns: A tuple of ``str`` containing (current_version, next_version) """ if version is None: # If no version is given, convert the build version number to a release version @@ -144,22 +125,12 @@ def get_version(version, installer_metadata): def get_installer_artifacts(run_id): """Download the artifacts from a GitHub Actions run - Parameters - ---------- - run_id: int - The GitHub Actions run ID to download artifacts from - - Returns - ------- - A tuple of the: - - * Git revision used for the given Run ID - * The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer - * The `bytes` corresponding to the DIRACOS2 installer itself - - Raises - ------ - A ``RuntimeError`` if the GitHub Actions run was not successful + :param int run_id: The GitHub Actions run ID to download artifacts from + :raises RuntimeError: The GitHub Actions run was not successful + :returns: A tuple of the: + * Git revision used for the given Run ID + * The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer + * The `bytes` corresponding to the DIRACOS2 installer itself """ r = requests.get(f"{api_root}/actions/runs/{run_id}", headers=headers) r.raise_for_status() @@ -191,17 +162,11 @@ def make_release(installer, environment_yaml, version, commit_hash): determined by PEP-440, it is promoted to at full release after the uploads have completed successfully. - Parameters - ---------- - installer : bytes - The `bytes` corresponding to the DIRACOS2 installer itself - environment_yaml : str - The `str` corresponding to the ``environment.yaml`` generated by the DIRACOS2 installer - version : str - The version of the new release - - commit_hash : str - DIRACOS2 git revision used for the release + :param bytes installer: The `bytes` corresponding to the DIRACOS2 installer itself + :param str environment_yaml: The `str` corresponding to the ``environment.yaml`` + generated by the DIRACOS2 installer + :param str version: The version of the new release + :param str commit_hash: DIRACOS2 git revision used for the release """ release_notes = "\n".join([ f"# DIRACOS {version}", @@ -291,10 +256,7 @@ def make_release(installer, environment_yaml, version, commit_hash): def bump_version_in_main(new_version): """Edit the construct.yaml file on main to correspond to the next DIRACOS2 version - Parameters - ---------- - new_version : str - The next version number of DIRACOS2 + :param str new_version: The next version number of DIRACOS2 """ r = requests.get(f"{api_root}/contents/construct.yaml", headers=headers) r.raise_for_status() From b469e0649ffbd90716b96007e118da83210ca353 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 9 Dec 2020 14:08:44 +0100 Subject: [PATCH 5/5] Mention scripts/make_release.py in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c862090..450e1d9 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ scripts/run_basic_tests.sh DOCKER_IMAGE DIRACOS_INSTALLER_FILENAME ### Making a release -To ensure reproducibility, releases are made from build artifacts from previous pipelines and are tagged using GitHub actions by triggering the [Create release](https://github.com/DIRACGrid/DIRACOS2/actions?query=workflow%3A%22Create+release%22) workflow. This workflow has the following optional parameters: +To ensure reproducibility, releases are made from build artifacts from previous pipelines and are tagged using GitHub actions by triggering the [Create release](https://github.com/DIRACGrid/DIRACOS2/actions?query=workflow%3A%22Create+release%22) workflow. This workflow runs [`scripts/make_release.py`](https://github.com/DIRACGrid/DIRACOS2/blob/main/scripts/make_release.py) and has the following optional parameters: * **Run ID**: The GitHub Actions workflow run ID. If not given, defaults to the most recent build of the `main` branch. * **Version number**: A [PEP-440](https://www.python.org/dev/peps/pep-0440/) compliant version number. If not given, defaults to the contents the contents of version field in the `construct.yaml` rounded to the next full release (i.e. `2.4a5` becomes `2.4` and `2.1` remains unchanged). If a pre-release is explicitly give, it will be marked as a pre-release in GitHub and won't affect the `latest` alias.