diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml index 7b79225ce..ed469a0b2 100644 --- a/.github/workflows/build_packages.yml +++ b/.github/workflows/build_packages.yml @@ -14,3 +14,6 @@ jobs: run: source tools/ci.sh && ci_build_packages_check_manifest - name: Compile package index run: source tools/ci.sh && ci_build_packages_compile_index + - name: Publish packages for branch + if: vars.MICROPY_PUBLISH_MIP_INDEX && github.event_name == 'push' && ! github.event.deleted + run: source tools/ci.sh && ci_push_package_index diff --git a/.github/workflows/cleanup_published_packages.yml b/.github/workflows/cleanup_published_packages.yml new file mode 100644 index 000000000..c6a33cece --- /dev/null +++ b/.github/workflows/cleanup_published_packages.yml @@ -0,0 +1,12 @@ +name: Cleanup published packages + +on: delete + +jobs: + cleanup: + runs-on: ubuntu-latest + if: vars.MICROPY_PUBLISH_MIP_INDEX + steps: + - uses: actions/checkout@v2 + - name: Clean up published files + run: source tools/ci.sh && ci_cleanup_package_index ${{ github.event.ref }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 715477171..804a26bef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,3 +69,49 @@ There are some specific conventions and guidelines for micropython-lib: * When porting an existing third-party package, please ensure that the source license is compatible. + +* To make it easier for others to install packages directly from your PR before + it is merged, consider opting-in to automatic package publishing (see + [Publishing packages from forks](#publishing-packages-from-forks)). If you do + this, consider quoting the [commands to install + packages](README.md#installing-packages-from-forks) in your Pull Request + description. + +### Publishing packages from forks + +You can easily publish the packages from your micropython-lib +[fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) +by opting in to a system based on [GitHub +Actions](https://docs.github.com/en/actions) and [GitHub +Pages](https://docs.github.com/en/pages): + +1. Open your fork's repository in the GitHub web interface. +2. Navigate to "Settings" -> "Secrets and variables" -> "Actions" -> "Variables". +3. Click "New repository variable" +4. Create a variable named `MICROPY_PUBLISH_MIP_INDEX` with value `true` (or any + "truthy" value). +5. The settings for GitHub Actions and GitHub Pages features should not need to + be changed from the repository defaults, unless you've explicitly disabled + them. + +The next time you push commits to a branch in your fork, GitHub Actions will run +an additional step in the "Build All Packages" workflow named "Publish Packages +for branch". + +Anyone can then install these packages as described under [Installing packages +from forks](README.md#installing-packages-from-forks). The exact commands are also +quoted in the GitHub Actions log for the "Publish Packages for branch" step. + +#### Opting Back Out + +To opt-out again, delete the `MICROPY_PUBLISH_MIP_INDEX` variable and +(optionally) delete the `gh-pages` branch from your fork. + +*Note*: While enabled, all micropython-lib packages will be published each time +a change is pushed to any branch in your fork. A commit is added to the +`gh-pages` branch each time. In a busy repository, the `gh-pages` branch may +become quite large. The actual `.git` directory size on disk should still be +quite small, as most of the content will be duplicated. If you're worried that +the `gh-pages` branch has become too large then you can always delete this +branch from GitHub. GitHub Actions will create a new `gh-pages` branch the next +time you push a change. diff --git a/README.md b/README.md index c47c0acf9..73417b965 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,37 @@ Note that unlike the other three approaches based on `mip` or `manifest.py`, you will need to manually resolve dependencies. You can inspect the relevant `manifest.py` file to view the list of dependencies for a given package. +## Installing packages from forks + +It is possible to use the `mpremote mip install` or `mip.install()` methods to +install packages built from a +[fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) +of micropython-lib, if the fork's owner has opted in. + +This can be useful to install packages from a pending Pull Request, for example. + +First, the owner of the fork must opt-in as described under +[Publishing packages from forks](CONTRIBUTING.md#publishing-packages-from-forks). + +After this has happened, each time someone pushes to a branch in that fork then +GitHub Actions will automatically publish the packages to a GitHub Pages site. + +To install these packages, use commands such as: + +```bash +$ mpremote connect /dev/ttyUSB0 mip install --index https://USERNAME.github.io/micropython-lib/mip/BRANCH_NAME PACKAGE_NAME +``` + +Or from a networked device: + +```py +import mip +mip.install(PACKAGE_NAME, index="https://USERNAME.github.io/micropython-lib/mip/BRANCH_NAME") +``` + +(Where `USERNAME`, `BRANCH_NAME` and `PACKAGE_NAME` are replaced with the owner +of the fork, the branch the packages were built from, and the package name.) + ## Contributing We use [GitHub Discussions](https://github.com/micropython/micropython/discussions) diff --git a/tools/ci.sh b/tools/ci.sh index 75c8791d5..548680682 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -1,5 +1,9 @@ #!/bin/bash +######################################################################################## +# common "constants" +PACKAGE_INDEX_PATH=/tmp/micropython-lib-deploy + ######################################################################################## # code formatting @@ -38,5 +42,104 @@ function ci_build_packages_check_manifest { } function ci_build_packages_compile_index { - python3 tools/build.py --micropython /tmp/micropython --output /tmp/micropython-lib-deploy + python3 tools/build.py --micropython /tmp/micropython --output $PACKAGE_INDEX_PATH +} + +function ci_push_package_index { + set -euo pipefail + + # Note: This feature is opt-in, so this function is only run by GitHub + # Actions if the MICROPY_PUBLISH_MIP_INDEX repository variable is set to a + # "truthy" value in the "Secrets and variables" -> "Actions" + # -> "Variables" setting of the GitHub repo. + + PAGES_PATH=/tmp/gh-pages + + if git fetch --depth=1 origin gh-pages; then + git worktree add ${PAGES_PATH} gh-pages + cd ${PAGES_PATH} + NEW_BRANCH=0 + else + echo "Creating gh-pages branch for $GITHUB_REPOSITORY..." + git worktree add --force ${PAGES_PATH} HEAD + cd ${PAGES_PATH} + git switch --orphan gh-pages + NEW_BRANCH=1 + fi + + DEST_PATH=${PAGES_PATH}/mip/${GITHUB_REF_NAME} + if [ -d ${DEST_PATH} ]; then + git rm -r ${DEST_PATH} + fi + mkdir -p ${DEST_PATH} + cd ${DEST_PATH} + + cp -r ${PACKAGE_INDEX_PATH}/* . + + git add . + git_bot_commit "Add CI built packages from commit ${GITHUB_SHA} of ${GITHUB_REF_NAME}" + + if [ "$NEW_BRANCH" -eq 0 ]; then + # A small race condition exists here if another CI job pushes to + # gh-pages at the same time, but this narrows the race to the time + # between these two commands. + git pull --rebase origin gh-pages + fi + git push origin gh-pages + + INDEX_URL="https://${GITHUB_REPOSITORY_OWNER}.github.io/$(echo ${GITHUB_REPOSITORY} | cut -d'/' -f2-)/mip/${GITHUB_REF_NAME}" + + echo "" + echo "--------------------------------------------------" + echo "Uploaded package files to GitHub Pages." + echo "" + echo "Unless GitHub Pages is disabled on this repo, these files can be installed remotely with:" + echo "" + echo "mpremote mip install --index ${INDEX_URL} PACKAGE_NAME" + echo "" + echo "or on the device as:" + echo "" + echo "import mip" + echo "mip.install(PACKAGE_NAME, index=\"${INDEX_URL}\")" +} + +function ci_cleanup_package_index() +{ + if ! git fetch --depth=1 origin gh-pages; then + exit 0 + fi + + # Argument $1 is github.event.ref, passed in from workflow file. + # + # this value seems to be a REF_NAME, without heads/ or tags/ prefix. (Can't + # use GITHUB_REF_NAME, this evaluates to the default branch.) + DELETED_REF="$1" + + if [ -z "$DELETED_REF" ]; then + echo "Bad DELETE_REF $DELETED_REF" + exit 1 # Internal error with ref format, better than removing all mip/ directory in a commit + fi + + # We need Actions to check out default branch and run tools/ci.sh, but then + # we switch branches + git switch gh-pages + + echo "Removing any published packages for ${DELETED_REF}..." + if [ -d mip/${DELETED_REF} ]; then + git rm -r mip/${DELETED_REF} + git_bot_commit "Remove CI built packages from deleted ${DELETED_REF}" + git pull --rebase origin gh-pages + git push origin gh-pages + else + echo "Nothing to remove." + fi +} + +# Make a git commit with bot authorship +# Argument $1 is the commit message +function git_bot_commit { + # Ref https://github.com/actions/checkout/discussions/479 + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "$1" }