From ed545379e27ec582d3d7281b6cbcb5315cf05a9f Mon Sep 17 00:00:00 2001 From: Robert Stoll Date: Mon, 26 Sep 2022 18:21:54 +0200 Subject: [PATCH] add .gitlab-ci.yml including script to install and to create MR moreover: - describe requirement of having PUBLIC_GPG_KEYS_WE_TRUST defined as secret in github - use branch gget/update instead of gget-update - also describe all requirements for the gitlab job --- .github/workflows/gget-update.yml | 8 +-- README.md | 80 +++++++++++++++++++++++++- scripts/cleanup-on-push-to-main.sh | 37 +++++++++--- src/gitlab/.gitlab-ci.yml | 45 +++++++++++++++ src/gitlab/create-mr.sh | 92 ++++++++++++++++++++++++++++++ src/gitlab/install-gget.sh | 51 +++++++++++++++++ src/gitlab/utils.sh | 41 +++++++++++++ 7 files changed, 341 insertions(+), 13 deletions(-) create mode 100644 src/gitlab/.gitlab-ci.yml create mode 100755 src/gitlab/create-mr.sh create mode 100755 src/gitlab/install-gget.sh create mode 100644 src/gitlab/utils.sh diff --git a/.github/workflows/gget-update.yml b/.github/workflows/gget-update.yml index f27ed117..29de5e9b 100644 --- a/.github/workflows/gget-update.yml +++ b/.github/workflows/gget-update.yml @@ -26,7 +26,7 @@ jobs: - name: Install gget run: | set -e - # see install.doc.sh in root of this project, MODIFY THERE NOT HERE + # see install.doc.sh in https://github.com/tegonal/gget, MODIFY THERE NOT HERE (please report bugs) currentDir=$(pwd) && \ tmpDir=$(mktemp -d -t gget-download-install-XXXXXXXXXX) && cd "$tmpDir" && \ wget "https://raw.githubusercontent.com/tegonal/gget/main/.gget/signing-key.public.asc" && \ @@ -50,11 +50,11 @@ jobs: - name: Create pull request if necessary uses: peter-evans/create-pull-request@v4 with: - branch: gget-update + branch: gget/update base: main - title: Updates via gget + title: Changes via gget update commit-message: update files pulled via gget - body: "following the changes after running: gget update" + body: "following the changes after running `gget update` (among other things)" delete-branch: true token: ${{ secrets.AUTO_PR_TOKEN }} push-to-fork: ${{ secrets.AUTO_PR_FORK_NAME }} diff --git a/README.md b/README.md index 483b5bbf..53d37380 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ For instance, the [README of v0.7.4](https://github.com/tegonal/gget/tree/v0.7.4 - [reset](#reset) - [update](#update) - [GitHub Workflow](#github-workflow) + - [Gitlab Job](#gitlab-job) - [self-update](#self-update) - [FAQ](#faq) - [Contributors and contribute](#contributors-and-contribute) @@ -672,6 +673,10 @@ This repository contains a github workflow which runs every week to check if the - the files which have been pulled as well as - the public keys of the remotes. +It requires you to define a variable named PUBLIC_GPG_KEYS_WE_TRUST which represents an armored export of all +gpg public keys you trust signing the public keys of remotes, +i.e. those are used to verify the public keys of the remotes you added via `gget remote add`. + You can re-use it in your repository. We suggest you fetch it via gget 😉 ```bash @@ -679,7 +684,12 @@ gget remote add -r gget -u https://github.com/tegonal/gget gget pull -r gget -p .github/workflows/gget-update.yml -d ./ ``` -Note thought, that it contains the following condition (so that it does not run in forks): +Accordingly, you would add [Tegonal's public key for github](https://tegonal.com/gpg/github.asc) to PUBLIC_GPG_KEYS_WE_TRUST in order +that this workflow can update the workflow itself. + +#### Required modifications + +You need to change one condition in the workflow which we added in order that this workflow does not run in forks: ```yml if: github.repository_owner == 'tegonal' @@ -707,6 +717,74 @@ function gget_pullHook_gget_after(){ Just make sure you replace YOUR_SLUG with your actual slug. +### Gitlab Job + +This repository contains a `.gitlab-ci.yml` which defines two job templates: +1. gget-update which checks if there are updates for: + - the files which have been pulled + - the public keys of the remotes + + and creates a Merge Request if there are some. + +2. gget-update-stop-pipeline which cancels itself and thus stops the pipeline + +You can re-use it in your repository. We suggest you fetch it via gget 😉 + +```bash +gget remote add -r gget -u https://github.com/tegonal/gget +gget pull -r gget -p src/gitlab/ +``` + +In your `.gitlab-ci.yml` you need to add `gget` to your stages and it should be the first stage: +```yml +stages: +- gget +... +``` +At some point you add in addition +```yml +include: 'lib/gget/src/gitlab/.gitlab-ci.yml' +``` + +That's it, this defines the two jobs. Yet, you need some extra configuration to be ready to use it... + +
+I need some modifications to the standard job + +If you need to run additional before_script or the like, then you can re-define +the job e.g. as follows (after the `include` above): +```yaml +gget-update: + extends: .gget-update + # your modifications here, e.g. for an additional step in before_script + before_script: + - !reference [.gget-update, before_script] + - cd subdirectory +``` + +
+ +#### Additional configuration + +The `gget-update` job (the `install-gget.sh` to be precise) +requires you to define a variable named PUBLIC_GPG_KEYS_WE_TRUST which represents an armored export of all +gpg public keys you trust signing the public keys of remotes, +i.e. those are used to verify the public keys of the remotes you added via `gget remote add`. + +For instance, if you fetched the gitlab job via gget as suggested, +then you would add [Tegonal's public key for github](https://tegonal.com/gpg/github.asc) +to PUBLIC_GPG_KEYS_WE_TRUST in order that this job can update itself. + +Moreover, the `create-mr.sh` requires an access token which is stored in variable GGET_UPDATE_API_TOKEN. +It is used to create the merge request. + +The gitlab job uses the image [gitlab-git](https://github.com/tegonal/gitlab-git) which requires you to define +the variable GITBOT_SSH_PRIVATE_KEY and a deploy key for it. +See [Basic Setup](https://github.com/tegonal/gitlab-git#basic-setup) for more information + +Now, all that is left is to create a scheduled pipeline (CI/CD -> Schedules) where you need to define Variable +`DO_GGET_UPDATE` with value `true`. Up to you how often you want to let it run (we run it weekly). + ## self-update You can update gget by using gget (which in turn uses its install.sh) diff --git a/scripts/cleanup-on-push-to-main.sh b/scripts/cleanup-on-push-to-main.sh index 6a07b57d..ee30be7f 100755 --- a/scripts/cleanup-on-push-to-main.sh +++ b/scripts/cleanup-on-push-to-main.sh @@ -39,16 +39,16 @@ function cleanupOnPushToMain() { -not -name "*.source.sh" \ -print0 | while read -r -d $'\0' script; do - declare relative + local relative relative="$(realpath --relative-to="$projectDir" "$script")" - declare id="${relative:4:-3}" + local id="${relative:4:-3}" updateBashDocumentation "$script" "${id////-}" . README.md || return $? replaceHelpSnippet "$script" "${id////-}-help" . README.md || return $? done || die "updating bash documentation and help snippets failed, see above" updateBashDocumentation "$projectDir/install.sh" "install" . README.md || die "could not update install documentation" - declare additionalHelp=( + local -ra additionalHelp=( gget_remote_add "src/gget-remote.sh" "add --help" gget_remote_remove "src/gget-remote.sh" "remove --help" gget_remote_list "src/gget-remote.sh" "list --help" @@ -59,13 +59,34 @@ function cleanupOnPushToMain() { replaceHelpSnippet "$projectDir/${additionalHelp[i + 1]}" "${additionalHelp[i]}-help" . README.md ${additionalHelp[i + 2]} done || die "replacing help snippets failed, see above" - local -r indent=' ' local installScript - installScript=$(perl -0777 -pe 's/(@|\$|\\)/\\$1/g;' < "$projectDir/install.doc.sh" | sed "s/^/$indent/" ) - perl -0777 -i -pe "s@(\n\s+# see install.doc.sh.*\n)[^#]+(# end install.doc.sh\n)@\${1}$installScript\n$indent\${2}@" \ - "$projectDir/.github/workflows/gget-update.yml" + installScript=$(perl -0777 -pe 's/(@|\$|\\)/\\$1/g;' <"$projectDir/install.doc.sh") - logSuccess "Updating bash docu and README completed" + local -ra includeInstallSh=( + "$projectDir/.github/workflows/gget-update.yml" 10 + "$projectDir/src/gitlab/install-gget.sh" 0 + ) + local -r arrLength="${#includeInstallSh[@]}" + local -i i + for ((i = 0; i < arrLength; i += 2)); do + local file="${includeInstallSh[i]}" + if ! [[ -f $file ]]; then + returnDying "file $file does not exist" || return $? + fi + + local indentNum="${includeInstallSh[i + 1]}" + local indent + indent=$(printf "%-${indentNum}s" "") || return $? + local content + # cannot use search/replace variable substitution + # shellcheck disable=SC2001 + content=$(sed "s/^/$indent/" <<<"$installScript") || return $? + perl -0777 -i \ + -pe "s@(\n\s+# see install.doc.sh.*\n)[^#]+(# end install.doc.sh\n)@\${1}$content\n$indent\${2}@" \ + "$file" || return $? + done || die "could not replace the install instructions" + + logSuccess "Cleanup on push to main completed" } ${__SOURCED__:+return} diff --git a/src/gitlab/.gitlab-ci.yml b/src/gitlab/.gitlab-ci.yml new file mode 100644 index 00000000..159fbbe9 --- /dev/null +++ b/src/gitlab/.gitlab-ci.yml @@ -0,0 +1,45 @@ +.gget-install: + script: &gget-install-script + - ./lib/gget/src/gitlab/install-gget.sh + +.gget-create-mr: + script: &gget-create-mr-script + - ./lib/gget/src/gitlab/create-mr.sh + +.gget-update: + stage: gget + image: tegonal/gitlab-git:latest + rules: + - if: $DO_GGET_UPDATE + variables: + GITBOT_USERNAME: 'gget bot' + GITBOT_EMAIL: 'gget@tegonal.com' + before_script: + - apk update && apk add bash git gnupg perl coreutils curl && apk upgrade + - tmpDir=$(mktemp -d -t gget-update-XXXXXXXXXX) && cd "$tmpDir" + - source /scripts/clone-current.sh + - export PATH="$PATH:$HOME/.local/bin" + script: + - *gget-install-script + - gget reset --gpg-only true + - gget update + - *gget-create-mr-script + +gget-update: + extends: .gget-update + +.gget-update-stop-pipeline: + stage: gget + image: alpine:latest + rules: + - if: $DO_GGET_UPDATE + needs: [ "gget-update" ] + script: + - apk update && apk add curl + - echo 'stopping the pipeline on purpose...' + - 'curl --request POST --header "PRIVATE-TOKEN: $GGET_UPDATE_API_TOKEN" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/cancel"' + - sleep + - echo 'cancel failed, stopping via exit...' + - exit 1 +gget-update-stop-pipeline: + extends: .gget-update-stop-pipeline diff --git a/src/gitlab/create-mr.sh b/src/gitlab/create-mr.sh new file mode 100755 index 00000000..be1af3e8 --- /dev/null +++ b/src/gitlab/create-mr.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# __ __ +# / /____ ___ ____ ___ ___ _/ / This script is provided to you by https://github.com/tegonal/gget +# / __/ -_) _ `/ _ \/ _ \/ _ `/ / It is licensed under Apache 2.0 +# \__/\__/\_, /\___/_//_/\_,_/_/ Please report bugs and contribute back your improvements +# /___/ +# Version: v0.8.0-SNAPSHOT +# +################################### +set -euo pipefail +shopt -s inherit_errexit +unset CDPATH + +if ! [[ -v dir_of_gget_gitlab ]]; then + dir_of_gget_gitlab="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" >/dev/null && pwd 2>/dev/null)" + readonly dir_of_gget_gitlab +fi +source "$dir_of_gget_gitlab/utils.sh" + +# is passed to exitIfEnvVarNotSet by name +# shellcheck disable=SC2034 +declare -a envVars=( + GGET_UPDATE_API_TOKEN + CI_API_V4_URL + CI_PROJECT_ID +) +exitIfEnvVarNotSet envVars +readonly GGET_UPDATE_API_TOKEN CI_API_V4_URL CI_PROJECT_ID + +declare gitStatus +gitStatus=$(git status --porcelain) || { + echo "the following command failed (see above): git status --porcelain" + exit 1 +} + +if [[ $gitStatus == "" ]]; then + echo "No git changes, i.e. no updates found, no need to create a merge request" + exit 0 +fi + +echo "Detected updates, going to push changes to branch gget/update" + +git branch -D "gget/update" 2 &>/dev/null || true +git checkout -b "gget/update" +git add . +git commit -m "Update files pulled via gget" +git push -f --set-upstream origin gget/update || { + echo "could not force push gget/update to origin" + exit 1 +} + +declare data +data=$( + # shellcheck disable=SC2312 + cat <<-EOM + { + "source_branch": "gget/update", + "target_branch": "main", + "title": "Changes via gget update", + "allow_collaboration": true, + "remove_source_branch": true + } + EOM +) + +echo "Going to create a merge request for the changes" + +curlOutputFile=$(mktemp -t "curl-output-XXXXXXXXXX") + +# passed by name to cleanupTmp +# shellcheck disable=SC2034 +readonly -a tmpPaths=(curlOutputFile) +trap 'cleanupTmp tmpPaths' EXIT + +statusCode=$( + curl --request POST \ + --header "PRIVATE-TOKEN: $GGET_UPDATE_API_TOKEN" \ + --data "$data" --header "Content-Type: application/json" \ + --output "$curlOutputFile" --write-out "%{response_code}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests" +) || { + echo "could not send the POST request for creating a merge request" + exit 1 +} +if [[ $statusCode = 409 ]] && grep "open merge request" "$curlOutputFile"; then + echo "There is already a merge request, no need to create another (we force pushed, so the MR is updated)" +elif [[ ! "$statusCode" == 2* ]]; then + printf "curl return http status code %s, expected 2xx. Message body:\n" "$statusCode" + cat "$curlOutputFile" + exit 1 +fi diff --git a/src/gitlab/install-gget.sh b/src/gitlab/install-gget.sh new file mode 100755 index 00000000..1d76e330 --- /dev/null +++ b/src/gitlab/install-gget.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# __ __ +# / /____ ___ ____ ___ ___ _/ / This script is provided to you by https://github.com/tegonal/gget +# / __/ -_) _ `/ _ \/ _ \/ _ `/ / It is licensed under Apache 2.0 +# \__/\__/\_, /\___/_//_/\_,_/_/ Please report bugs and contribute back your improvements +# /___/ +# Version: v0.8.0-SNAPSHOT +# +################################### +set -euo pipefail +shopt -s inherit_errexit +unset CDPATH +if ! [[ -v dir_of_gget_gitlab ]]; then + dir_of_gget_gitlab="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" >/dev/null && pwd 2>/dev/null)" + readonly dir_of_gget_gitlab +fi +source "$dir_of_gget_gitlab/utils.sh" + +# is passed to exitIfEnvVarNotSet by name +# shellcheck disable=SC2034 +declare -a envVars=( + PUBLIC_GPG_KEYS_WE_TRUST +) +exitIfEnvVarNotSet envVars +readonly PUBLIC_GPG_KEYS_WE_TRUST + +# passed by name to cleanupTmp +# shellcheck disable=SC2034 +readonly -a tmpPaths=(tmpDir) +trap 'cleanupTmp tmpPaths' EXIT + +gpg --import - <<<"$PUBLIC_GPG_KEYS_WE_TRUST" + +# see install.doc.sh in https://github.com/tegonal/gget, MODIFY THERE NOT HERE (please report bugs) +currentDir=$(pwd) && \ +tmpDir=$(mktemp -d -t gget-download-install-XXXXXXXXXX) && cd "$tmpDir" && \ +wget "https://raw.githubusercontent.com/tegonal/gget/main/.gget/signing-key.public.asc" && \ +wget "https://raw.githubusercontent.com/tegonal/gget/main/.gget/signing-key.public.asc.sig" && \ +gpg --verify ./signing-key.public.asc.sig ./signing-key.public.asc && \ +echo "public key trusted" && \ +mkdir ./gpg && \ +gpg --homedir ./gpg --import ./signing-key.public.asc && \ +wget "https://raw.githubusercontent.com/tegonal/gget/main/install.sh" && \ +wget "https://raw.githubusercontent.com/tegonal/gget/main/install.sh.sig" && \ +gpg --homedir ./gpg --verify ./install.sh.sig ./install.sh && \ +chmod +x ./install.sh && \ +echo "verification successful" || (echo "verification failed, don't continue"; exit 1) && \ +./install.sh && result=true || (echo "installation failed"; exit 1) && \ +false || cd "$currentDir" && rm -r "$tmpDir" && "${result:-false}" +# end install.doc.sh diff --git a/src/gitlab/utils.sh b/src/gitlab/utils.sh new file mode 100644 index 00000000..140955aa --- /dev/null +++ b/src/gitlab/utils.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# __ __ +# / /____ ___ ____ ___ ___ _/ / This script is provided to you by https://github.com/tegonal/gget +# / __/ -_) _ `/ _ \/ _ \/ _ `/ / It is licensed under Apache 2.0 +# \__/\__/\_, /\___/_//_/\_,_/_/ Please report bugs and contribute back your improvements +# /___/ +# Version: v0.8.0-SNAPSHOT +# +################################### +set -euo pipefail +shopt -s inherit_errexit +unset CDPATH +export GGET_VERSION='v0.8.0-SNAPSHOT' + +function exitIfEnvVarNotSet() { + local -rn exitIfEnvVarNotSet_arr=$1 + shift 1 + + declare error=false + for envName in "${exitIfEnvVarNotSet_arr[@]}"; do + if ! [[ -v "$envName" ]] || [[ -z ${!envName} ]]; then + echo "Looks like you forgot to define the variable $envName" + error=true + fi + done + if [[ $error == true ]]; then + echo "In GitLab, go to Settings => CI/CD => Variables and define it/them there" + echo "See also https://github.com/tegonal/gget/tree/${GGET_VERSION}#gitlab-job for further information" + exit 1 + fi +} + +function cleanupTmp() { + local -rn cleanupTmp_paths=$1 + for path in "${cleanupTmp_paths[@]}"; do + if [[ -v "$path" && -n ${!path} ]]; then + rm -rf "$path" + fi + done +}