Nightlies #1481
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightlies | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - actions | |
| pull_request: | |
| branches: | |
| - '*' | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| setup: | |
| name: 'build settings' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| settings: ${{ steps.settings.outputs.settings }} | |
| deploy: ${{ steps.settings.outputs.deploy }} | |
| env: | |
| # Bump this value when a rebuild of nightlies (of the | |
| # same compiler commits) has to be done. | |
| nightlies_revision: 4 | |
| steps: | |
| - name: Checkout nightlies | |
| uses: actions/checkout@v4 | |
| with: | |
| path: nightlies | |
| - name: Generate version matrix | |
| shell: bash | |
| run: | | |
| # Tracked branches | |
| branches=( 'devel' 'version-2-0' 'version-1-6') | |
| getHash() { | |
| git ls-remote "https://github.com/$1" "$2" | cut -f 1 | |
| } | |
| { | |
| for branch in "${branches[@]}"; do | |
| jq --null-input \ | |
| --arg branch "$branch" \ | |
| --arg commit "$(getHash nim-lang/Nim "$branch")" \ | |
| '{ branch: $branch, commit: $commit }' | |
| done | |
| } | jq -s '.' | tee versions.json | |
| - name: Restore build settings | |
| uses: actions/cache@v4 | |
| with: | |
| path: settings | |
| key: build-settings-${{ hashFiles('versions.json') }}-${{ env.nightlies_revision }} | |
| restore-keys: build-settings- | |
| - name: Generate build settings | |
| id: settings | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| deploy=false | |
| case '${{ github.event_name }}' in | |
| schedule) | |
| deploy=true | |
| ;; | |
| push) | |
| message=$( | |
| cd nightlies | |
| git show -s --format=%s '${{ github.ref }}' | |
| ) | |
| if [[ $message == *'[deploy]'* ]]; then | |
| deploy=true | |
| else | |
| deploy=false | |
| fi | |
| esac | |
| declare -A refs | |
| while IFS='=' read -r branch commit; do | |
| refs[$branch]=$commit | |
| done <<< "$(jq -r '.[] | .branch + "=" + .commit' versions.json)" | |
| mkdir -p settings | |
| # Delete older branch settings (if they were restored) | |
| find settings -mindepth 1 -maxdepth 1 $(printf "! -name %q.json " "${!refs[@]}") | |
| declare -A environment=( | |
| [SOURCE_CODE_EPOCH]=$(date -u +%s) | |
| ) | |
| for branch in "${!refs[@]}"; do | |
| savedCommit= | |
| savedNightliesRev= | |
| savedBuildRevision= | |
| commit=${refs[$branch]} | |
| buildRevision=0 | |
| if [[ -e "settings/$branch.json" ]]; then | |
| savedSettings=$(< "settings/$branch.json") | |
| savedCommit=$(jq -r '.commit' <<< "$savedSettings") | |
| savedNightliesRev=$(jq '.nightlies_revision // 0' <<< "$savedSettings") | |
| savedBuildRevision=$(jq '.build_revision // -1' <<< "$savedSettings") | |
| if [[ $savedNightliesRev -ne $nightlies_revision ]]; then | |
| buildRevision=$(($savedBuildRevision + 1)) | |
| fi | |
| fi | |
| if [[ $savedCommit != "$commit" || $savedNightliesRev -ne $nightlies_revision ]]; then | |
| echo "::group::Generating build settings for branch $branch" | |
| { | |
| { | |
| for var in "${!environment[@]}"; do | |
| jq --null-input \ | |
| --arg variable "$var" \ | |
| --arg value "${environment[$var]}" \ | |
| '{ ($variable): $value }' | |
| done | |
| } | jq -s '{ environment: (reduce .[] as $item (null; . + $item)) }' | |
| tag=$(date -u --date="@${environment[SOURCE_CODE_EPOCH]}" +%F)-$branch-$commit | |
| # Don't add revision to tags for now. | |
| # if [[ $buildRevision -gt 0 ]]; then | |
| # tag=$tag-$buildRevision | |
| # fi | |
| jq --null-input --arg tag "$tag" '{ release: $tag }' | |
| jq --null-input --argjson nightlies_revision "$nightlies_revision" '{ nightlies_revision: $nightlies_revision }' | |
| jq --null-input --argjson build_revision "$buildRevision" '{ build_revision: $build_revision }' | |
| if [[ "$tag" == version-2-0 ]]; then | |
| jq --null-input --argjson csources "nim-lang/csources_v2" '{ csources: $csources }' | |
| fi | |
| if [[ "$tag" == devel ]]; then | |
| jq --null-input --argjson csources "nim-lang/csources_v2" '{ csources: $csources }' | |
| fi | |
| } | jq -s --arg commit "$commit" 'reduce .[] as $item ({ commit: $commit }; . + $item)' > "settings/$branch.json" | |
| else | |
| echo "::group::Stored settings for branch $branch @ $commit found." | |
| fi | |
| jq --arg branch "$branch" '.[] | select(.branch == $branch)' versions.json | jq -s 'add' - "settings/$branch.json" | tee -a settings.json | |
| echo "::endgroup::" | |
| done | |
| echo "settings=$(jq -sc '.' settings.json)" >> $GITHUB_OUTPUT | |
| echo "deploy=$deploy" >> $GITHUB_OUTPUT | |
| sourceArchive: | |
| needs: setup | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| setting: ${{ fromJson(needs.setup.outputs.settings) }} | |
| name: 'source (${{ matrix.setting.branch }}, ${{ matrix.setting.commit }})' | |
| runs-on: ubuntu-latest | |
| env: ${{ matrix.setting.environment }} | |
| steps: | |
| - name: Checkout nightlies | |
| uses: actions/checkout@v4 | |
| with: | |
| path: nightlies | |
| - name: 'Install dependencies' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y hub | |
| - name: Restore Nim from cache | |
| id: nim-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: nim/output/nim-*.tar.xz | |
| key: 'source-${{ matrix.setting.commit }}-${{ matrix.setting.build_revision }}' | |
| - name: Checkout Nim | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: nim-lang/Nim | |
| ref: ${{ matrix.setting.commit }} | |
| path: nim | |
| - name: Get csources version | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| id: csources-version | |
| shell: bash | |
| run: | | |
| csources_repo=${{ matrix.setting.csources }} | |
| # if one is not set, use csources_v1 | |
| : "${csources_repo:=nim-lang/csources_v1}" | |
| csources_commit=$(git ls-remote "https://github.com/$csources_repo" master | cut -f 1) | |
| if [[ -f nim/config/build_config.txt ]]; then | |
| . nim/config/build_config.txt | |
| csources_repo=${nim_csourcesUrl#https://github.com/} | |
| csources_repo=${csources_repo%.git} | |
| csources_commit=$nim_csourcesHash | |
| fi | |
| echo "repo=$csources_repo" >> $GITHUB_OUTPUT | |
| echo "commit=$csources_commit" >> $GITHUB_OUTPUT | |
| - name: Restore csources from cache | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| id: csources-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: csources/bin | |
| key: > | |
| csources-${{ runner.os }}-${{ steps.csources-version.outputs.repo }}-${{ steps.csources-version.outputs.commit }} | |
| - name: Checkout csources | |
| if: > | |
| steps.nim-cache.outputs.cache-hit != 'true' && | |
| steps.csources-cache.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ steps.csources-version.outputs.repo }} | |
| ref: ${{ steps.csources-version.outputs.commit }} | |
| path: csources | |
| - name: Setup environment | |
| shell: bash | |
| run: echo '${{ github.workspace }}/nim/bin' >> "$GITHUB_PATH" | |
| - name: Build 1-stage csources compiler | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| if [[ ! -e csources/bin/nim ]]; then | |
| make -C csources -j $(nproc) CC=gcc | |
| else | |
| echo 'Using prebuilt csources' | |
| fi | |
| cp csources/bin/nim nim/bin | |
| - name: Build compiler | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| cd nim | |
| nim c koch | |
| ./koch boot -d:release | |
| - name: Generate csources | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| cd nim | |
| ./koch csources -d:danger '-d:gitHash:${{ matrix.setting.commit }}' | |
| - name: Bundle external sources | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| cd nim | |
| version=$(nim secret --hints:off <<< 'echo NimVersion; quit 0') | |
| major=${version%%.*} | |
| minor=${version#*.} | |
| minor=${minor%.*} | |
| patch=${version##*.} | |
| ./koch nimble | |
| # Only Nim >= 1.4.0 bundles fusion. | |
| if [[ $major -ge 1 && $minor -ge 4 ]]; then | |
| ./koch fusion | |
| fi | |
| - name: Build source archive | |
| if: steps.nim-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| cd nim | |
| ./koch xz | |
| mkdir -p output | |
| source=$(basename build/nim-*.tar.xz) | |
| version=${source%.tar.xz} | |
| version=${version#nim-} | |
| cp build/$source output/nim-$version.tar.xz | |
| - name: Publish release | |
| if: needs.setup.outputs.deploy == 'true' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if ! hub -C nightlies release show '${{ matrix.setting.release }}' >/dev/null 2>&1; then | |
| cat << EOF | hub -C nightlies release create -a nim/output/* -F - '${{ matrix.setting.release }}' >/dev/null | |
| Nightly build on $(date -u --date="@$SOURCE_CODE_EPOCH" "+%F") for branch ${{ matrix.setting.branch }} | |
| Commit: https://github.com/nim-lang/Nim/commit/${{ matrix.setting.commit }} | |
| Generated release binaries will be uploaded as they're made available. | |
| EOF | |
| else | |
| echo "Release '${{ matrix.setting.release }}' has already been created, skipping." | |
| fi | |
| - name: Upload source archive to artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: 'nim-${{ matrix.setting.commit }}' | |
| path: nim/output/* | |
| build: | |
| needs: [ setup, sourceArchive ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| setting: ${{ fromJson(needs.setup.outputs.settings) }} | |
| target: | |
| - os: linux | |
| triple: x86_64-linux-musl | |
| - os: linux | |
| triple: i686-linux-musl | |
| - os: linux | |
| triple: aarch64-linux-musl | |
| - os: linux | |
| triple: armv7l-linux-musleabihf | |
| - os: macosx | |
| triple: x86_64-apple-darwin14 | |
| - os: windows | |
| triple: x86_64-w64-mingw32 | |
| - os: windows | |
| triple: i686-w64-mingw32 | |
| include: | |
| - target: | |
| os: linux | |
| builder: ubuntu-20.04 | |
| - target: | |
| os: macosx | |
| builder: macos-11 | |
| - target: | |
| os: windows | |
| builder: windows-2019 | |
| env: ${{ matrix.setting.environment }} | |
| name: '${{ matrix.target.triple }} (${{ matrix.setting.branch }}, ${{ matrix.setting.commit }})' | |
| runs-on: ${{ matrix.builder }} | |
| steps: | |
| - name: Checkout build scripts | |
| uses: actions/checkout@v4 | |
| with: | |
| path: nightlies | |
| - name: Cache build outputs | |
| id: built | |
| uses: actions/cache@v4 | |
| with: | |
| path: output | |
| key: > | |
| output-${{ hashFiles('nightlies/lib.sh') }}-${{ hashFiles('nightlies/build-release.sh') }}-${{ matrix.target.triple }}-${{ matrix.setting.commit }}-${{ matrix.setting.build_revision }} | |
| - name: Cache dependencies | |
| if: steps.built.outputs.cache-hit != 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: external | |
| key: > | |
| deps-${{ hashFiles('nightlies/lib.sh') }}-${{ hashFiles('nightlies/deps.sh') }}-${{ hashFiles('nightlies/buildreq.txt') }}-${{ runner.os }}-${{ matrix.target.triple }}-${{ matrix.setting.build_revision }} | |
| - name: Install dependencies | |
| if: steps.built.outputs.cache-hit != 'true' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: nightlies/deps.sh -t '${{ matrix.target.triple }}' | |
| - name: Download generated source package | |
| if: steps.built.outputs.cache-hit != 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: 'nim-${{ matrix.setting.commit }}' | |
| path: source | |
| - name: Extract source package | |
| if: steps.built.outputs.cache-hit != 'true' | |
| id: source | |
| shell: bash | |
| run: | | |
| source=( source/nim-*.tar.xz ) | |
| version=${source[0]##*nim-} | |
| version=${version%%.tar.xz} | |
| case '${{ runner.os }}' in | |
| 'Windows') | |
| 7z x -so "${source[0]}" | 7z x -si -ttar -aoa | |
| ;; | |
| *) | |
| tar xJf "${source[0]}" | |
| ;; | |
| esac | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| - name: Setup environment | |
| if: steps.built.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| echo "-d:gitHash:\"${{ matrix.setting.commit }}\"" >> external/nim.cfg | |
| - name: Build release binaries | |
| if: steps.built.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| nightlies/build-release.sh 'nim-${{ steps.source.outputs.version }}' | |
| - name: Prepare binaries for uploads | |
| id: release | |
| shell: bash | |
| run: | | |
| source nightlies/lib.sh | |
| artifact=$(< output/nim.txt) | |
| echo "artifact=$artifact" >> $GITHUB_OUTPUT | |
| # Github Actions work based on native Windows path, so we're doing | |
| # some quick conversions here. | |
| echo "artifact_nativepath=$(nativepath "$artifact")" >> $GITHUB_OUTPUT | |
| - name: Upload release binaries | |
| if: needs.setup.outputs.deploy == 'true' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd nightlies | |
| if ! gh release view '${{ matrix.setting.release }}' | grep "$(basename '${{ steps.release.outputs.artifact }}')" >/dev/null 2>&1; then | |
| gh release upload '${{ matrix.setting.release }}' '${{ steps.release.outputs.artifact }}' | |
| else | |
| echo "Binary already released for tag '${{ matrix.setting.release }}', not overwritting." | |
| fi | |
| - name: Upload binaries to artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: 'binaries-${{ matrix.setting.branch }}-${{ matrix.setting.commit }}' | |
| path: '${{ steps.release.outputs.artifact_nativepath }}' | |
| latestTag: | |
| needs: [ setup, build ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| setting: ${{ fromJson(needs.setup.outputs.settings) }} | |
| if: needs.setup.outputs.deploy == 'true' | |
| name: 'Update latest tags for ${{ matrix.setting.release }}' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Store build settings | |
| shell: bash | |
| run: | | |
| cat <<< '${{ toJson(matrix.setting) }}' > setting.json | |
| - name: Check whether deployment was done for this setting set | |
| id: deploy-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: setting.json | |
| key: deploy-${{ hashFiles('setting.json') }}-${{ env.nightlies_revision }} | |
| - name: Checkout nightlies | |
| if: steps.deploy-cache.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| path: nightlies | |
| - name: Download generated source package | |
| if: steps.deploy-cache.outputs.cache-hit != 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: 'nim-${{ matrix.setting.commit }}' | |
| path: source | |
| - name: Download built binaries from artifacts | |
| if: steps.deploy-cache.outputs.cache-hit != 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: 'binaries-${{ matrix.setting.branch }}-${{ matrix.setting.commit }}' | |
| path: binaries | |
| - name: 'Push latest-${{ matrix.setting.branch }} tag' | |
| if: steps.deploy-cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| tag='latest-${{ matrix.setting.branch }}' | |
| mkdir -p assets | |
| source=("$GITHUB_WORKSPACE"/source/*) | |
| # Rename the source archive as source.tar.xz | |
| mv -v "${source[0]}" "assets/source.tar.xz" | |
| for artifact in binaries/*; do | |
| # Trim version from artifact name | |
| shortName=${artifact#*nim-*-} | |
| mv -v "$artifact" "assets/$shortName" | |
| done | |
| cat << EOF > release-notes.md | |
| This release is an alias for the latest successful nightly build \ | |
| for branch \`${{ matrix.setting.branch }}\`: https://github.com/${{ github.repository }}/releases/tag/${{ matrix.setting.release }} | |
| Each \`latest-\` alias is guaranteed to contain binaries for all \ | |
| supported architectures. | |
| EOF | |
| cd nightlies | |
| # By deleting and recreating the tag, Github recognize it as the true | |
| # "latest" tags. | |
| echo "Deleting release $tag (errors are ignored)" | |
| gh release delete -y "$tag" || : | |
| git push --delete origin "$tag" || : | |
| echo "Publishing $tag" | |
| gh release create \ | |
| -t 'Latest successful build for branch ${{ matrix.setting.branch }}' \ | |
| -F "$GITHUB_WORKSPACE/release-notes.md" \ | |
| "$tag" \ | |
| "$GITHUB_WORKSPACE/assets"/* |