build-ubuntu.yaml

name: Build git-annex on Ubuntu

on:
  # Trigger the workflow on pull requests
  pull_request:
    paths:
      - '.github/workflows/build-ubuntu.yaml'
      - 'patches/*.patch'
  schedule:
    - cron: '30 02 * * *'
  workflow_dispatch:
    inputs:
      commitish:
        description: The upstream commitish to build
      pr:
        description: The number of the PR to build

defaults:
  run:
    shell: bash

env:
  LANG: C
  DEB_SIGN_KEYID: 13A1093296154584245E0300C98FC49D36DAB17F
  DEB_BUILD_OPTIONS: nocheck
  bbuild_log: git-annex-build.log

jobs:
  build-package:
    runs-on: ubuntu-latest
    outputs:
      build-version: ${{ steps.build-version.outputs.version }}
    steps:
      - name: Checkout this repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Create pending PR status
        if: github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            build-package \
            pending
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up system
        run: |
          bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh)
          sudo apt-get update -qq
          sudo apt-get install eatmydata
          sudo eatmydata apt-get install gnupg moreutils strace

      - name: Install Singularity
        run: |
          # use if decide to get most recent release of singularity
          # release="$(curl -fsSL https://api.github.com/repos/sylabs/singularity/releases/latest | jq -r .tag_name)"
          release="v3.10.4"
          codename="$(lsb_release -cs)"
          arch="$(dpkg --print-architecture)"
          cd /tmp
          wget -O singularity-ce.deb https://github.com/sylabs/singularity/releases/download/$release/singularity-ce_${release#v}-${codename}_$arch.deb
          sudo dpkg -i singularity-ce.deb
          sudo apt-get install -f

      - name: Determine git-annex ref to build
        run: |
          . .github/workflows/tools/set-build-commit
          echo "BUILD_COMMIT=$BUILD_COMMIT" >> "$GITHUB_ENV"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          INPUT_PR: ${{ github.event.inputs.pr }}
          INPUT_COMMITISH: ${{ github.event.inputs.commitish }}

      - name: Check out source files
        run: |
          # The goal here is for $BUILD_COMMIT to be the HEAD (necessary for
          # git-annex's version detection to use the correct git commit) with
          # the contents of master — or whatever ref is being used as the
          # workflow source — also available.
          git checkout "$BUILD_COMMIT"
          git checkout "$GITHUB_SHA" -- .

      - name: Get build version
        id: build-version
        run: |
          version="$(git describe "$BUILD_COMMIT" | sed -e 's/-/+git/')"
          arch="$(dpkg --print-architecture)"
          echo "Building $version"
          echo "version=${version}_$arch" >> "$GITHUB_OUTPUT"

      - name: Apply local patches
        run: |
          .github/workflows/tools/apply-patches patches ${{ github.event_name }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Adjust changelog generation script to use original branch
        run: sed -i -e "s,HEAD,$BUILD_COMMIT,g" debian/create-standalone-changelog

      - name: Import GPG keys
        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false
        run: |
          # private key for signing - comes from secrets
          echo '${{ secrets.datalad_builder_gpgkey }}' | gpg --import
          # public key for verification
          gpg --import .github/workflows/tools/datalad-builder-key.asc

      - name: Get the git-annex build environment container
        run: chronic singularity pull --name buildenv.sif docker://datalad/buildenv-git-annex
        working-directory: /tmp

      - name: Build source packages
        run: |
          chronic singularity exec /tmp/buildenv.sif make debianstandalone-dsc
          mv ../git-annex_*.* .

      - name: Build binary package from the source package
        run: |
          sign="${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}"
          if [ "x$sign" = xtrue ]
          then opts=
          else opts='-uc -us'
          fi
          if ! singularity exec /tmp/buildenv.sif \
               bash -c "set -x; export LANG=C; dpkg-source -x *.dsc git-annex-source && cd git-annex-source && dpkg-buildpackage -b $opts" >| "${bbuild_log}" 2>&1; then
                 echo "E: build failed"
                 echo "I: the tail of the log"
                 tail -n 50 "${bbuild_log}"
                 echo "I: FAIL line matches with some context"
                 grep -B10 '^FAIL$' "${bbuild_log}"
                 exit 1
          fi

      - name: Check that all tests were run and passed
        run: grep -E '^All [[:digit:]]{3} tests passed' "${bbuild_log}"
        if: "!contains(env.DEB_BUILD_OPTIONS, 'nocheck')"

      - name: Upload packages
        uses: actions/upload-artifact@v3
        with:
          name: git-annex-debianstandalone-packages_${{ steps.build-version.outputs.version }}
          path: |
            git-annex[-_]*.*
            dist/build-version

      - name: Clone datalad/git-annex-ci-client-jobs
        if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
        uses: actions/checkout@v3
        with:
          repository: datalad/git-annex-ci-client-jobs
          fetch-depth: 1
          path: client-jobs
          ssh-key: ${{ secrets.CLIENT_JOBS_SSH_KEY }}

      - name: Push installer to datalad/git-annex-ci-client-jobs
        if: contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
        run: |
          set -ex -o pipefail
          git checkout --orphan build
          git rm -rf .
          cp -i ../git-annex*.deb .
          git add git-annex*.deb
          buildno="${{ github.run_number }}"
          git commit \
            --gpg-sign=13A1093296154584245E0300C98FC49D36DAB17F \
            -m "Installer artifact from build $buildno"
          git ls-remote --heads origin "build-*" | cut -f2 | cut -d/ -f3- > builds.txt
          git ls-remote --heads origin "result-*" | cut -f2 | cut -d/ -f3- > results.txt
          yq -r "keys | .[]" ../clients/clients.yaml | while read clientid
          do build_branch="build-$clientid-$buildno"
             result_branch="result-$clientid-$buildno"
             if grep -Fqx "$result_branch" results.txt
             then echo "[INFO] Result branch $result_branch already exists; not creating build branch"
             elif grep -Fqx "$build_branch" builds.txt
             then echo "[INFO] Build branch $build_branch already exists; leaving alone"
             else git branch "$build_branch" build
                  git push origin "$build_branch"
             fi
          done
        working-directory: client-jobs

      - name: Create new release
        if: github.event.inputs.commitish != ''
        run: |
          printf '[DEBUG] INPUT_COMMITISH=%b\n' "$INPUT_COMMITISH"
          if git rev-parse refs/tags/"$INPUT_COMMITISH" &> /dev/null
          then
              echo "[INFO] Building a tag; uploading assets to release ..."
              echo '[DEBUG] BEGIN gh release list'
              gh release list | tee releases.txt | cat -v
              echo '[DEBUG] END gh release list'
              if grep -q "^$INPUT_COMMITISH\b" releases.txt
              then
                  echo "[INFO] Release already exists; uploading assets"
                  gh release upload "$INPUT_COMMITISH" git-annex[-_]*.*
              else
                  echo "[INFO] Creating release"
                  gh release create \
                    --notes "This is an unofficial release build provided by the DataLad team." \
                    "$INPUT_COMMITISH" git-annex[-_]*.*
              fi
          else
              echo "[INFO] Not building a tag; no release to make"
          fi
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          INPUT_COMMITISH: ${{ github.event.inputs.commitish }}

      - name: Set final PR status
        if: always() && github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            build-package \
            "${{ job.status }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Send e-mail on failed run
        if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
        uses: dawidd6/action-send-mail@v3
        with:
          server_address: ${{ secrets.NOTIFY_SMTP_HOST }}
          server_port: ${{ secrets.NOTIFY_SMTP_PORT }}
          username: ${{ secrets.NOTIFY_SMTP_USERNAME }}
          password: ${{ secrets.NOTIFY_SMTP_PASSWORD }}
          from: GitHub Actions Notifications
          to: ${{ secrets.NOTIFY_RECIPIENT }}
          subject: '[${{ github.repository }}] Build on Ubuntu failed!'
          body: |
            A build (via ${{ github.event_name }}) of git-annex for Ubuntu failed!

            See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.

  test-annex:
    runs-on: ${{ matrix.os }}
    needs: build-package
    strategy:
      matrix:
        flavor: ["normal", "crippled-tmp", "crippled-home", "nfs-home", "custom-config1"]
        os: [ubuntu-latest]
      fail-fast: false
    steps:
      - name: Checkout this repository
        uses: actions/checkout@v3

      - name: Create pending PR status
        if: github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            "test-annex (${{ matrix.flavor }}, ${{ matrix.os }})" \
            pending
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download git-annex package
        uses: actions/download-artifact@v3
        with:
          name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }}

      - name: Install git-annex package
        run: |
          sudo dpkg -i git-annex*.deb

      - name: Print git-annex version
        run: git annex version

      - name: Run tests
        run: |
          if echo "${{ matrix.flavor }}" | grep -q "crippled" ; then
            # source so that CRIPPLEDFS_PATH will be set:
            . .github/workflows/tools/setup_crippledfs crippledfs 500
          fi
          if echo "${{ matrix.flavor }}" | grep -q "nfs" ; then
            mkdir /tmp/nfsmount_ /tmp/nfsmount
            echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports'
            sudo apt-get install -y nfs-kernel-server
            sudo exportfs -a
            sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount
          fi

          case "${{ matrix.flavor }}" in
            # For git-annex it causes only few temporary directories to be on the crippled FS,
            # while the main ones produced by git annex test  reside in CWD, for which we use
            # $HOME
            crippled-tmp)
              export TMPDIR="$CRIPPLEDFS_PATH"
              ;;
            nfs-tmp)
              export TMPDIR=/tmp/nfsmount
              ;;
            # Also used as CWD where running the tests, so in effect tests annex operating
            # on those file systems
            crippled-home)
              export HOME="$CRIPPLEDFS_PATH"
              ;;
            nfs-home)
              export HOME=/tmp/nfsmount
              ;;
            normal|custom-config1)
              ;;
            *)
              echo "Unknown flavor ${{ matrix.flavor }}"
              exit 1
          esac

          # Do it after we possibly setup HOME
          git config --global user.email "test@github.land"
          git config --global user.name "GitHub Almighty"
          test_opts=( )

          case "${{ matrix.flavor }}" in
            # For git-annex it causes only few temporary directories to be on the crippled FS,
            # while the main ones produced by git annex test  reside in CWD, for which we use
            # $HOME
            custom-config1)
              test_opts=( --test-git-config annex.stalldetection=1KB/120s )
              ;;
            *)
              ;;
          esac

          cd $HOME
          export | grep -e crippledfs || :

          timeout 3600 git annex test "${test_opts[@]:-}"

      - name: Set final PR status
        if: always() && github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            "test-annex (${{ matrix.flavor }}, ${{ matrix.os }})" \
            "${{ job.status }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Send e-mail on failed run
        if: failure() && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
        uses: dawidd6/action-send-mail@v3
        with:
          server_address: ${{ secrets.NOTIFY_SMTP_HOST }}
          server_port: ${{ secrets.NOTIFY_SMTP_PORT }}
          username: ${{ secrets.NOTIFY_SMTP_USERNAME }}
          password: ${{ secrets.NOTIFY_SMTP_PASSWORD }}
          from: GitHub Actions Notifications
          to: ${{ secrets.NOTIFY_RECIPIENT }}
          subject: '[${{ github.repository }}] Tests of Ubuntu build failed!'
          body: |
            The tests for a build (via ${{ github.event_name }}) of git-annex for Ubuntu (flavor: ${{ matrix.flavor }}, OS: ${{ matrix.os }}) failed!

            See <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}> for more information.

  test-annex-more:
    runs-on: ubuntu-latest
    needs: build-package
    steps:
      - name: Checkout this repository
        uses: actions/checkout@v3

      - name: Create pending PR status
        if: github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            test-annex-more \
            pending
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download git-annex package
        uses: actions/download-artifact@v3
        with:
          name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }}

      - name: Install git-annex package
        run: |
          sudo dpkg -i git-annex*.deb

      - name: Seek of dynlibs
        run: |
          mkdir /tmp/testrepo; cd /tmp/testrepo; git init
          function nfailed() {
            strace -f git-annex "$1" 2>&1 | awk "/$2.*ENOENT/{print}" | tee /dev/fd/2 | wc -l
          }
          # We should get some reasonable number (not 40) of directories look up for dynamic libraries
          liblookups=
          PS4='> '; set -x
          test $(nfailed version "libpcre.*so") -lt 7
          test $(nfailed init "libpcre.*so") -lt 260

      - name: Set final PR status
        if: always() && github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            test-annex-more \
            "${{ job.status }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  test-datalad:
    runs-on: ubuntu-latest
    needs: build-package
    strategy:
      matrix:
        version: [master, maint, release]
      fail-fast: false
    steps:
      - name: Checkout this repository
        uses: actions/checkout@v3

      - name: Create pending PR status
        if: github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            "test-datalad (${{ matrix.version }})" \
            pending
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download git-annex package
        uses: actions/download-artifact@v3
        with:
          name: git-annex-debianstandalone-packages_${{ needs.build-package.outputs.build-version }}

      - name: Install git-annex package
        run: |
          sudo dpkg -i git-annex*.deb

      - name: Set up SSH target
        shell: bash
        run: |
          curl -fSsL \
            https://raw.githubusercontent.com/datalad/datalad/master/tools/ci/prep-travis-forssh.sh \
            | bash
          echo DATALAD_TESTS_SSH=1 >> "$GITHUB_ENV"


      - name: Set up environment
        run: |
          git config --global user.email "test@github.land"
          git config --global user.name "GitHub Almighty"

      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install ${{ matrix.version }} Datalad
        run: |
          if [ "${{ matrix.version }}" = "release" ]; then
            # Authorize so that we don't run up against low API rate limits
            commitish="$(curl -fsSL -H "Authorization: bearer ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/datalad/datalad/releases/latest | jq -r .tag_name)"
          else
            commitish="${{ matrix.version }}"
          fi
          python -m pip install --upgrade pip
          pip install git+https://github.com/datalad/datalad@${commitish}

      - name: Install nose et al (just in case!)
        run: pip install nose pytest vcrpy mock

      - name: WTF!?
        run: datalad wtf

      # needed for ssh certs under ubuntu and tox.ini everywhere
      - name: Checkout datalad
        uses: actions/checkout@v3
        with:
          repository: datalad/datalad
          path: datalad
          fetch-depth: 1

      - name: Set up test SSH certs
        run: bash tools/ci/deploy_datalad-rootca
        working-directory: datalad

      - name: Run datalad tests
        run: |
          mkdir -p __testhome__
          cd __testhome__
          python -m pytest -c ../datalad/tox.ini -s -v --pyargs datalad

      - name: Set final PR status
        if: always() && github.event.inputs.pr != ''
        run: |
          .github/workflows/tools/set-pr-status \
            "${{ github.event.inputs.pr }}" \
            Ubuntu \
            "test-datalad (${{ matrix.version }})" \
            "${{ job.status }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# vim:set et sts=2: