diff --git a/update_submodule_versions/Dockerfile b/update_submodule_versions/Dockerfile new file mode 100644 index 0000000..8b699fb --- /dev/null +++ b/update_submodule_versions/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10-bullseye + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +COPY requirements.txt /tmp/ + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y git && \ + pip3 install --upgrade pip && \ + pip3 install -r /tmp/requirements.txt + +COPY entrypoint.sh / +COPY update_submodule_versions.py / + +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] diff --git a/update_submodule_versions/README.md b/update_submodule_versions/README.md new file mode 100644 index 0000000..c059903 --- /dev/null +++ b/update_submodule_versions/README.md @@ -0,0 +1,33 @@ +# Update repository submodules action + +This action helps automate updates to submodules of a repository. It is similar to Dependabot's submodule update functionality, with a few extra features: + +1. Configuration of this action, specific to each submodule, is stored along with the rest of submodule information in `.gitmodules` file. +2. The action updates the submodule to the latest tag matching a certain pattern on a given branch. +3. The action can optionally update idf_component.yml file to the version matching the upstream version. + +## Configuration + +This action reads configuration from custom options in `.gitmodules` file. Here is an example: +``` +[submodule "fmt/fmt"] + path = fmt/fmt + url = https://github.com/fmtlib/fmt.git + autoupdate = true + autoupdate-branch = master + autoupdate-tag-glob = [0-9]*.[0-9]*.[0-9]* + autoupdate-include-lightweight = true + autoupdate-manifest = fmt/idf_component.yml + autoupdate-ver-regex = ([0-9]+).([0-9]+).([0-9]+) +``` + + +| Option | Possible values | Default | Explanation | +|--------------------------------|---------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------| +| autoupdate | `true`, `false` | `false` | Whether to update this submodule or not | +| autoupdate-branch | string | | Name of the submodule branch where to look for the new tags. Required if autoupdate=true. | +| autoupdate-tag-glob | Git glob expression | | Glob pattern (as used by 'git describe --match') to use when looking for tags. Required if autoupdate=true. | +| autoupdate-include-lightweight | `true`, `false` | `false` | Whether to include lightweight (not annotated) tags. | +| autoupdate-manifest | path relative to Git repository | | If specified, sets the name of the idf_component.yml file where the version should be updated. | +| autoupdate-ver-regex | regular expression | | Regular expression to extract major, minor, patch version numbers from the Git tag. Required if autoupdate-manifest is set. | + diff --git a/update_submodule_versions/action.yml b/update_submodule_versions/action.yml new file mode 100644 index 0000000..7460870 --- /dev/null +++ b/update_submodule_versions/action.yml @@ -0,0 +1,21 @@ +name: "Update submodules" +description: "Make PRs to update submodules to new release tags" +inputs: + repo-token: + description: "Github API token (for opening PRs)" + required: true + git-author-name: + description: "Commit author name" + required: true + git-author-email: + description: "Commit author email" + required: true +runs: + using: "docker" + image: "Dockerfile" + env: + GITHUB_TOKEN: ${{ inputs.repo-token }} + GIT_AUTHOR_NAME: ${{ inputs.git-author-name }} + GIT_AUTHOR_EMAIL: ${{ inputs.git-author-email }} + GIT_COMMITTER_NAME: ${{ inputs.git-author-name }} + GIT_COMMITTER_EMAIL: ${{ inputs.git-author-email }} diff --git a/update_submodule_versions/entrypoint.sh b/update_submodule_versions/entrypoint.sh new file mode 100644 index 0000000..e3ff605 --- /dev/null +++ b/update_submodule_versions/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +git config --global --add safe.directory "*" + +/usr/local/bin/python3 /update_submodule_versions.py \ + --repo ${GITHUB_WORKSPACE} \ + --open-github-pr-in ${GITHUB_REPOSITORY} \ + + + diff --git a/update_submodule_versions/requirements.txt b/update_submodule_versions/requirements.txt new file mode 100644 index 0000000..50cf1c1 --- /dev/null +++ b/update_submodule_versions/requirements.txt @@ -0,0 +1,3 @@ +GitPython==3.1.29 +ruamel.yaml==0.17.21 +PyGithub==1.58.1 diff --git a/update_submodule_versions/test_update_submodule_versions.py b/update_submodule_versions/test_update_submodule_versions.py new file mode 100644 index 0000000..bb0af8a --- /dev/null +++ b/update_submodule_versions/test_update_submodule_versions.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import tempfile +import textwrap +import unittest + +from git import Repo, Commit + +from update_submodule_versions import * + + +class UpdateSubmoduleVersionsTest(unittest.TestCase): + def setUp(self) -> None: + # create the repo for a dependency + self.dependency_dir = Path(tempfile.mkdtemp()) + self.dependency_repo = Repo.init(self.dependency_dir) + + # add a file and make the first commit + dependency_readme_file = self.dependency_dir / "README.md" + dependency_readme_file.write_text("This is a dependency\n") + self.dependency_repo.index.add([dependency_readme_file.name]) + dep_commit = self.dependency_repo.index.commit( + "initial commit of the dependency" + ) + self.dependency_repo.create_head("main", commit=dep_commit.hexsha) + + # create the "project" repo where the submodule will be added + self.project_dir = Path(tempfile.mkdtemp()) + self.project_repo = Repo.init(self.project_dir.absolute()) + + # add the dependency as a submodule and commit it + self.submodule = self.project_repo.create_submodule( + "dependency", "dependency", url=self.dependency_dir, branch="main" + ) + self.project_repo.index.commit("added a dependency as a submodule") + + self.addCleanup(self.dependency_dir) + self.addCleanup(self.project_dir) + + def create_commit(self, repo: Repo, filename: str, commit_msg: str) -> Commit: + """Make a commit in the given repo, creating an empty file""" + file_path = Path(repo.working_tree_dir) / filename + file_path.touch() + repo.index.add([filename]) + return repo.index.commit(message=commit_msg) + + def tag_dependency(self, tag_name: str) -> Commit: + """Make a commit in the dependency and tag it with the given name""" + dep_commit = self.create_commit( + self.dependency_repo, f"release_{tag_name}.md", f"Release {tag_name}" + ) + self.dependency_repo.create_tag( + tag_name, dep_commit.hexsha, message=f"Release {tag_name}" + ) + return dep_commit + + def update_dependency_submodule_to(self, commit: Commit, commit_msg: str): + submodule = self.project_repo.submodule("dependency") + submodule.binsha = commit.binsha + submodule.update() + self.project_repo.index.add([submodule]) + self.project_repo.index.commit(commit_msg) + + def test_can_update_manually(self): + """This is just a test to check that the setUp and above functions work okay""" + self.create_commit(self.dependency_repo, "1.txt", "Added 1.txt") + submodule_commit = self.tag_dependency("v1.0") + self.update_dependency_submodule_to( + submodule_commit, "update submodule to v1.0" + ) + self.assertTrue((self.project_dir / "dependency" / "1.txt").exists()) + self.assertEqual( + "v1.0", + self.project_repo.git.submodule("--quiet foreach git describe".split()), + ) + + def test_find_latest_remote_tag(self): + """Check that find_latest_remote_tag function finds the tagged commit""" + + # Create a tag, check that it is found on the right commit + first_commit = self.create_commit(self.dependency_repo, "1.txt", "Added 1.txt") + self.create_commit(self.dependency_repo, "2.txt", "Added 2.txt") + v2_release_commit = self.tag_dependency("v2.0") + self.create_commit(self.dependency_repo, "3.txt", "Added 3.txt") + tag_found = find_latest_remote_tag(self.submodule, "main", "v*") + self.assertEqual(v2_release_commit.hexsha, tag_found.commit.hexsha) + + # Create a tag on an older commit, check that the most recent tag + # (in branch sequential order) is found, not the most recent one + # in chronological order + self.dependency_repo.create_tag( + "v1.0", first_commit.hexsha, message=f"Release v1.0" + ) + tag_found = find_latest_remote_tag(self.submodule, "main", "v*") + self.assertEqual(v2_release_commit.hexsha, tag_found.commit.hexsha) + + # Check that the wildcard is respected, by looking specifically for v1* tags + tag_found = find_latest_remote_tag(self.submodule, "main", "v1*") + self.assertEqual(first_commit.hexsha, tag_found.commit.hexsha) + + # Create a newer tag on another branch, check that it is not found + self.dependency_repo.create_head( + "release/v2.0", commit=v2_release_commit.hexsha + ) + self.dependency_repo.git.checkout("release/v2.0") + self.create_commit(self.dependency_repo, "2_1.txt", "Added 2_1.txt") + v2_1_release_commit = self.tag_dependency("v2.1") + + tag_found = find_latest_remote_tag(self.submodule, "main", "v*") + self.assertEqual(v2_release_commit.hexsha, tag_found.commit.hexsha) + + # But the newest tag should be found if we specify the release branch + tag_found = find_latest_remote_tag(self.submodule, "release/v2.0", "v*") + self.assertEqual(v2_1_release_commit.hexsha, tag_found.commit.hexsha) + + +class VersionFromTagTest(unittest.TestCase): + def test_version_from_tag(self): + self.assertEqual( + IdfComponentVersion(1, 2, 3), + get_version_from_tag("v1.2.3", DEFAULT_TAG_VERSION_REGEX), + ) + self.assertEqual( + IdfComponentVersion(1, 2, 3), + get_version_from_tag("1.2.3", DEFAULT_TAG_VERSION_REGEX), + ) + self.assertEqual( + IdfComponentVersion(1, 2, 0), + get_version_from_tag("1.2", DEFAULT_TAG_VERSION_REGEX), + ) + self.assertEqual( + IdfComponentVersion(2, 4, 9), + get_version_from_tag("R_2_4_9", r"R_(\d+)_(\d+)_(\d+)"), + ) + + with self.assertRaises(ValueError): + get_version_from_tag("v1.2.3-rc1", DEFAULT_TAG_VERSION_REGEX) + with self.assertRaises(ValueError): + get_version_from_tag("qa-test-v1.2.3", DEFAULT_TAG_VERSION_REGEX) + with self.assertRaises(ValueError): + get_version_from_tag("v1.2.3.4", DEFAULT_TAG_VERSION_REGEX) + with self.assertRaises(ValueError): + get_version_from_tag("v1", DEFAULT_TAG_VERSION_REGEX) + + +class UpdateIDFComponentYMLVersionTest(unittest.TestCase): + def update_manifest(self, orig_yaml: str, new_ver: IdfComponentVersion): + with tempfile.NamedTemporaryFile("a+") as manifest_file: + manifest_file.write(orig_yaml) + manifest_file.flush() + update_idf_component_yml_version(Path(manifest_file.name), new_ver) + manifest_file.seek(0) + return manifest_file.read() + + def test_update_manifest_version(self): + self.assertEqual( + textwrap.dedent( + """ + # this is a comment + version: "2.0.1" + """ + ), + self.update_manifest( + textwrap.dedent( + """ + # this is a comment + version: "1.2.0" + """ + ), + IdfComponentVersion(2, 0, 1), + ), + ) + + self.assertEqual( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "2.0.2" + """ + ), + self.update_manifest( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "2.0.1~1" + """ + ), + IdfComponentVersion(2, 0, 2), + ), + ) + + self.assertEqual( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "4.3.1" + """ + ), + self.update_manifest( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "4.3.1~1-rc.1" + """ + ), + IdfComponentVersion(4, 3, 1), + ), + ) + + with self.assertRaises(ValueError): + self.update_manifest( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + # no version tag + """ + ), + IdfComponentVersion(1, 0, 0), + ) + + with self.assertRaises(ValueError): + self.update_manifest( + textwrap.dedent( + """ + version: "0.1.0" + repository: "https://github.com/espressif/idf-extra-components.git" + version: "0.1.1" + """ + ), + IdfComponentVersion(1, 0, 0), + ) + + self.assertEqual( + textwrap.dedent( + """ + # version: "1.0.0" + version: "2.0.1" + """ + ), + self.update_manifest( + textwrap.dedent( + """ + # version: "1.0.0" + version: "1.2.0" + """ + ), + IdfComponentVersion(2, 0, 1), + ), + ) + + self.assertEqual( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "2.0.1" # trailing comment + """ + ), + self.update_manifest( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "1.2.0" # trailing comment + """ + ), + IdfComponentVersion(2, 0, 1), + ), + ) + + # check that we add a newline in case version is on the last line and + # the line was missing a newline + self.assertEqual( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "2.0.1" # no newline + """ + ), + self.update_manifest( + textwrap.dedent( + """ + repository: "https://github.com/espressif/idf-extra-components.git" + version: "1.2.0" # no newline""" + ), + IdfComponentVersion(2, 0, 1), + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/update_submodule_versions/update_submodule_versions.py b/update_submodule_versions/update_submodule_versions.py new file mode 100644 index 0000000..45db65e --- /dev/null +++ b/update_submodule_versions/update_submodule_versions.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python +# +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# +import argparse +import collections +import contextlib +import logging +import os +import re +import sys +import typing +from pathlib import Path + +import git +import github + +DEFAULT_TAG_VERSION_REGEX = r"v?(\d+)\.(\d+)(?:\.(\d+))?$" +REMOTE = "origin" + +IdfComponentVersion = collections.namedtuple( + "IdfComponentVersion", ["major", "minor", "patch"] +) + +SubmoduleConfig = collections.namedtuple( + "SubmoduleConfig", + [ + "submodule", + "branch", + "tag_glob", + "include_lightweight", + "manifest", + "ver_regex", + "url", + ], +) + + +def find_latest_remote_tag( + repo: git.Repo, branch: str, tag_glob: str, include_lightweight: bool +) -> git.TagReference: + """ + Find the latest tag which matches the given glob pattern on the remote branch + """ + remote_branch = f"{REMOTE}/{branch}" + describe_args = [ + "--abbrev=0", + remote_branch, + ] + if tag_glob: + describe_args += ["--match", tag_glob] + if include_lightweight: + describe_args += ["--tags"] + try: + latest_tag_name = repo.git.describe(describe_args) + except git.GitCommandError as e: + raise RuntimeError( + f"Failed to run 'git describe {' '.join(describe_args)}': {e.stderr}" + ) + return repo.tag(latest_tag_name) + + +def is_commit_ahead_of_tag( + repo: git.repo, commit: git.Commit, tag: git.TagReference +) -> bool: + """ + Returns true if "commit" is more recent than "tag" + """ + try: + repo.git.describe(commit.hexsha, tags=True, match=tag.name) + return True + except git.GitCommandError: + return False + + +def update_submodule_pointer( + repo: git.repo, submodule: git.Submodule, new_tag: git.TagReference +) -> None: + """ + Change the submodule pointer to the specified tag and add this change to the index + of the parent repository. + """ + submodule.binsha = new_tag.commit.binsha + submodule.update() + repo.index.add([submodule]) + + +def finish_commit( + repo: git.Repo, submodule: git.Submodule, new_tag_name: str, commit_desc: str +) -> None: + """ + When changes have already been staged, this function creates a commit describing the + upgrade to the new version. + + """ + commit_msg = f"{submodule.path}: Update to {new_tag_name}\n\n{commit_desc}" + repo.index.commit(commit_msg) + logging.info(f'Created commit: "{commit_msg}"') + + +def get_version_from_tag(tag_name: str, version_regex: str) -> IdfComponentVersion: + match = re.match(version_regex, tag_name) + if not match: + raise ValueError( + f'Tag "{tag_name}" doesn\'t match version regex "{version_regex}"' + ) + + match_groups_num = len(list(filter(lambda g: g is not None, match.groups()))) + if match_groups_num == 2: + patch = 0 + elif match_groups_num == 3: + patch = int(match.group(3)) + else: + raise ValueError( + f"Regular expression must have 2 or 3 match groups, got {match_groups_num}" + ) + + return IdfComponentVersion( + major=int(match.group(1)), minor=int(match.group(2)), patch=patch + ) + + +def update_idf_component_yml_version(idf_cmp_yml: Path, ver: IdfComponentVersion): + """Rewrite the version value in the specified idf_component.yml file with the given version""" + with open(idf_cmp_yml, "r", encoding="utf-8") as f: + lines = f.readlines() + + version_regex = re.compile( + r'^(?P
version\s*:\s*)"[^"]+"(?P