build.yml

name: Build & test
concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - main
    tags:
    - '*'
  pull_request:
    branches:
      - main

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  # Build installers
  build:
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: macos-11
            arch: x86_64

          - os: macos-11
            arch: arm64

          - os: ubuntu-20.04
            arch: x86_64

          - os: windows-2019
            arch: x86_64

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -el {0}

    steps:
      # Based on https://docs.github.com/en/actions/deployment/deploying-xcode-applications/installing-an-apple-certificate-on-macos-runners-for-xcode-development
      - name: Install code-signing certificates (macOS)
        if: ${{ runner.os == 'macOS' && github.event_name != 'pull_request' }}
        env:
          APPLICATION_CERT_BASE64: ${{ secrets.APPLE_APPLICATION_CERT_BASE64 }}
          APPLICATION_CERT_PASSWORD: ${{ secrets.APPLE_APPLICATION_CERT_PASSWORD }}
          INSTALLER_CERT_BASE64: ${{ secrets.APPLE_INSTALLER_CERT_BASE64 }}
          INSTALLER_CERT_PASSWORD: ${{ secrets.APPLE_INSTALLER_CERT_PASSWORD }}
          KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
        run: |
          # create variables
          APPLICATION_CERT_PATH=$RUNNER_TEMP/application_cert.p12
          INSTALLER_CERT_PATH=$RUNNER_TEMP/installer_cert.p12
          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db

          # import certificatefrom secrets
          echo -n "$APPLICATION_CERT_BASE64" | base64 --decode --output $APPLICATION_CERT_PATH
          echo -n "$INSTALLER_CERT_BASE64" | base64 --decode --output $INSTALLER_CERT_PATH

          # create temporary keychain
          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

          # download Apple certificates
          curl https://www.apple.com/appleca/AppleIncRootCertificate.cer -L --output AppleIncRootCertificate.cer
          curl https://www.apple.com/certificateauthority/AppleComputerRootCertificate.cer -L --output AppleComputerRootCertificate.cer
          curl http://developer.apple.com/certificationauthority/AppleWWDRCA.cer -L --output AppleWWDRCA.cer

          # install Apple certificates
          # the following line is required for macOS 11+, see
          # https://developer.apple.com/forums/thread/671582?answerId=693632022#693632022
          sudo security authorizationdb write com.apple.trust-settings.admin allow
          sudo security add-trusted-cert -d -r trustRoot -k $KEYCHAIN_PATH ./AppleIncRootCertificate.cer
          sudo security add-trusted-cert -d -r trustRoot -k $KEYCHAIN_PATH ./AppleComputerRootCertificate.cer
          security add-certificates -k $KEYCHAIN_PATH ./AppleWWDRCA.cer

          # ensure we're going to import the correct developer certificates into keychain
          openssl pkcs12 -nokeys -passin pass:"$APPLICATION_CERT_PASSWORD" -in $APPLICATION_CERT_PATH | grep friendlyName
          openssl pkcs12 -nokeys -passin pass:"$INSTALLER_CERT_PASSWORD" -in $INSTALLER_CERT_PATH | grep friendlyName

          # import developer certificates
          security import $APPLICATION_CERT_PATH -P "$APPLICATION_CERT_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
          security import $INSTALLER_CERT_PATH -P "$INSTALLER_CERT_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH

          # ensure the imported certificates are valid
          security find-identity -v $KEYCHAIN_PATH

          # Avoid a password prompt; what this actually does is not properly
          # documented; see https://github.com/fastlane/fastlane/issues/13564#issue-372273249
          # and https://stackoverflow.com/a/40039594
          # FIXME: Really needed?
          security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

          # Make the keychain the default
          security default-keychain -s $KEYCHAIN_PATH

          # List available signing identities (for debugging purposes)
          security find-identity

      - uses: actions/checkout@v3

      - name: Install Micromamba
        uses: mamba-org/provision-with-micromamba@main
        with:
          environment-file: environment.yml
          micromamba-version: latest

      - name: Hotfix for NSIS
        run: |
          mamba install nsis=3.01 conda=4.12a "pydantic<2.0a0"
        if: ${{ runner.os == 'Windows' }}

      - name: Extract version information
        env:
          MNE_CROSSCOMPILE_ARCH: ${{ matrix.arch }}
        run: |
          source ./tools/extract_version.sh
          echo "Version:     ${MNE_INSTALLER_VERSION} (Python=${PYSHORT})"
          test "$MNE_INSTALLER_VERSION" != ""
          test "$PYSHORT" != ""
          test -d "$SCRIPT_DIR"
          echo "Recipe:      ${RECIPE_DIR}"
          test "$RECIPE_DIR" != ""
          test -d "$RECIPE_DIR"
          echo "Installer:   ${MNE_INSTALLER_NAME}"
          test "$MNE_INSTALLER_NAME" != ""
          echo "Artifact ID: ${MNE_INSTALLER_ARTIFACT_ID}"
          test "$MNE_INSTALLER_ARTIFACT_ID" != ""
          echo "MNE_INSTALLER_VERSION=${MNE_INSTALLER_VERSION}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_NAME=${MNE_INSTALLER_NAME}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_ARTIFACT_ID=${MNE_INSTALLER_ARTIFACT_ID}" >> $GITHUB_ENV
          echo "RECIPE_DIR=${RECIPE_DIR}" >> $GITHUB_ENV

      - name: Patch constructor
        run: |
          ./tools/patch_constructor.sh

      - name: Add macOS M1 support
        if: ${{ matrix.arch == 'arm64' }}
        run: |
          source ./tools/setup_m1_crosscompile.sh
          echo "EXE_ARG=${EXE_ARG}" >> $GITHUB_ENV
          echo "PLATFORM_ARG=${PLATFORM_ARG}" >> $GITHUB_ENV

      - name: Patch config (macOS pull request)
        if: ${{ runner.os == 'macOS' && github.event_name != 'pull_request' }}
        run: |
          sed -i "" "s/_name: *# \[osx\]/_name: 9779L28NP8  # \[osx\]/" ${RECIPE_DIR}/construct.yaml

      - name: Build installer
        # As of 2022/03/14, ~7 min on 20.04 (fastest) and ~11 min on windows-2019 (slowest).
        # So let's set this to a reasonable limit that will tell us more quickly
        # if something has gone wrong with dependency resolution.
        #
        # Update 2022/06/03: Bump timeout as we're seeing sporadic issues on
        #                    Linux
        timeout-minutes: 50
        run: |
          ./tools/run_constructor.sh

      - name: Check installer signature (macOS)
        if: ${{ runner.os == 'macOS' && github.event_name != 'pull_request' }}
        run: |
          # Installer package
          pkgutil --check-signature ${MNE_INSTALLER_NAME}
          # Now extract the package and check that the _conde.exe binary is
          # properly signed as well
          pkgutil --expand-full ${MNE_INSTALLER_NAME} ./mne-extracted
          codesign -vd ./mne-extracted/main.pkg/Payload/.mne-python/_conda.exe
          # Check entitlements of _conda.exe binary
          codesign --display --entitlements - ./mne-extracted/main.pkg/Payload/.mne-python/_conda.exe
          rm -rf ./mne-extracted

      - name: Notarize installer (macOS)
        if: ${{ matrix.os == 'macOS-11' && github.event_name != 'pull_request' }}
        env:
          APPLE_ID: ${{ secrets.APPLE_ID }}
          APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
        run: |
          # Notarize the installer
          xcrun notarytool submit ./${MNE_INSTALLER_NAME} \
              --wait \
              --apple-id=$APPLE_ID \
              --password=$APPLE_ID_PASSWORD \
              --team-id=$APPLE_TEAM_ID
          # Staple the notarization certificate onto it
          xcrun stapler staple ${MNE_INSTALLER_NAME}

      # Note: if this changes, it needs to be changed in the Windows signing step as well
      - name: Calculate SHA256 hash of installer package
        run: |
          shopt -s nullglob  # Fail if the following pattern yields no results
          echo "Finding matches"
          matches=(MNE-Python-*-*.*)
          echo "Extracting fname"
          installer_fname="${matches[0]}"
          echo "Found name: ${installer_fname}"
          echo "Want name:  ${MNE_INSTALLER_NAME}"
          test "$installer_fname" == "$MNE_INSTALLER_NAME"
          hash_fname="${MNE_INSTALLER_NAME}.sha256.txt"
          shasum -a 256 "$MNE_INSTALLER_NAME" > "$hash_fname"
          cat "$hash_fname"

      - name: Check installer (macOS)
        if: ${{ runner.os == 'macOS' }}
        run: |
          installer -verbose -pkginfo -pkg ./${MNE_INSTALLER_NAME}
          installer -verbose -dominfo -pkg ./${MNE_INSTALLER_NAME}
          installer -verbose -volinfo -pkg ./${MNE_INSTALLER_NAME}

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.MNE_INSTALLER_ARTIFACT_ID }}
          path: MNE-Python-*.*

  # The Windows signing needs to be done on Linux
  # https://www.ssl.com/how-to/cloud-code-signing-integration-with-github-actions/
  sign:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Determine installer name (cross-OS)
        if: github.event_name != 'pull_request'
        run: |
          source ./tools/extract_version.sh Windows
          echo "MNE_INSTALLER_ARTIFACT_ID=${MNE_INSTALLER_ARTIFACT_ID}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_NAME=${MNE_INSTALLER_NAME}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_VERSION=${MNE_INSTALLER_VERSION}" >> $GITHUB_ENV
          mkdir signed
      - name: Download appropriate installer
        if: github.event_name != 'pull_request'
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.MNE_INSTALLER_ARTIFACT_ID }}
      - name: EV-code sign installer (Windows)
        if: github.event_name != 'pull_request'
        uses: sslcom/esigner-codesign@develop
        with:
          command: sign
          username: ${{ secrets.ES_USERNAME }}
          password: ${{ secrets.ES_PASSWORD }}
          credential_id: ${{ secrets.ES_CREDENTIAL_ID }}
          totp_secret: ${{ secrets.ES_TOTP_SECRET }}
          file_path: ${GITHUB_WORKSPACE}/${{ env.MNE_INSTALLER_NAME }}
          override: true
      - name: Calculate new SHA256 hash of installer package
        if: github.event_name != 'pull_request'
        run: |
          shopt -s nullglob  # Fail if the following pattern yields no results
          echo "Finding matches"
          matches=(MNE-Python-*-*.*)
          echo "Extracting fname"
          installer_fname="${matches[0]}"
          echo "Found name: ${installer_fname}"
          echo "Want name:  ${MNE_INSTALLER_NAME}"
          test "$installer_fname" == "$MNE_INSTALLER_NAME"
          hash_fname="${MNE_INSTALLER_NAME}.sha256.txt"
          echo "Old hash:"
          cat "$hash_fname"
          OLD_HASH="$(cat $hash_fname)"
          shasum -a 256 "$MNE_INSTALLER_NAME" > "$hash_fname"
          echo "New hash:"
          cat "$hash_fname"
          test "$(cat $hash_fname)" != "$OLD_HASH"
      - name: Overwrite artifacts
        if: github.event_name != 'pull_request'
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.MNE_INSTALLER_ARTIFACT_ID }}
          path: MNE-Python-*.*

  # Test
  test:
    needs: [build, sign]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-20.04, ubuntu-22.04, macos-11, macos-12, windows-2019, windows-2022]
        arch: [x86_64]
        # We currently can't/don't test anything about the arm64 build
        # include:
        #   - os: macos-11
        #     arch: arm64

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -el {0}

    steps:
      - uses: actions/checkout@v3

      - name: Determine installer name
        run: |
          source ./tools/extract_version.sh
          echo "MNE_INSTALLER_ARTIFACT_ID=${MNE_INSTALLER_ARTIFACT_ID}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_ARTIFACT_ID=${MNE_INSTALLER_ARTIFACT_ID}"
          echo "MNE_INSTALLER_NAME=${MNE_INSTALLER_NAME}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_NAME=${MNE_INSTALLER_NAME}"
          echo "MNE_INSTALLER_VERSION=${MNE_INSTALLER_VERSION}" >> $GITHUB_ENV
          echo "MNE_INSTALLER_VERSION=${MNE_INSTALLER_VERSION}"
          echo "MNE_ACTIVATE=${MNE_ACTIVATE}" >> $GITHUB_ENV
          echo "MNE_ACTIVATE=${MNE_ACTIVATE}"

      - name: Download appropriate installer
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.MNE_INSTALLER_ARTIFACT_ID }}

      - name: Check installer signing (Windows)
        if: ${{ runner.os == 'Windows' && github.event_name != 'pull_request' }}
        shell: cmd
        run: |
          "C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe" verify /pa /v .\%MNE_INSTALLER_NAME%

      - name: Run installer (macOS Intel)
        if: ${{ runner.os == 'macOS' && matrix.arch == 'x86_64' }}
        run: |
          sudo installer \
            -verbose \
            -pkg ${MNE_INSTALLER_NAME} \
            -target / \
            || ( tail -n 30 /var/log/install.log  && exit 1 ) # display last log messages on error

      - name: Run installer (Linux)
        if: ${{ runner.os == 'Linux' }}
        run: |
          echo `pwd`
          sh ./${MNE_INSTALLER_NAME} -b

      # https://docs.anaconda.com/anaconda/install/silent-mode.html#windows
      - name: Run installer (Windows)
        if: ${{ runner.os == 'Windows' }}
        timeout-minutes: 20
        shell: cmd
        run: |
          .\%MNE_INSTALLER_NAME% /S /InstallationType=JustMe /AddToPath=1

      - name: Export frozen environment definition
        if: ${{ matrix.arch == 'x86_64' }}
        run: |
          source "${MNE_ACTIVATE}"
          mamba list --json > ${MNE_INSTALLER_NAME}.env.json
          cat ${MNE_INSTALLER_NAME}.env.json

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ env.MNE_INSTALLER_ARTIFACT_ID }}-json
          path: MNE-Python-*.env.json

      - name: Setup graphical support (Linux)
        if: ${{ runner.os == 'Linux' }}
        run: |
          source "${MNE_ACTIVATE}"
          sudo apt-get install -y xvfb
          /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1280x1024x24 -ac +extension GLX +render -noreset -nolisten tcp -nolisten unix
          export DISPLAY=":99"
          echo "DISPLAY=:99" >> $GITHUB_ENV

      - name: Setup graphical support (Windows)
        if: ${{ runner.os == 'Windows' }}
        run: |
          git clone --depth 1 https://github.com/pyvista/gl-ci-helpers.git
          powershell ./gl-ci-helpers/appveyor/install_opengl.ps1

      - name: Check installation
        if: ${{ matrix.arch == 'x86_64' }}
        run: |
          source "${MNE_ACTIVATE}"
          conda info
          mamba list

          if [[ "${{ runner.os }}" == "macOS" ]]; then
            echo "Testing that file permissions are set correctly (owned by "runner", not "root".)"
            # https://unix.stackexchange.com/a/7733
            [ `ls -ld /Applications/MNE-Python/.mne-python | awk 'NR==1 {print $3}'` == "runner" ] || exit 1
            # Check that the installed Python is, in fact, an Intel binary
            python -c "import platform; assert platform.machine() == 'x86_64'" || exit 1
            echo "Checking we have all .app bundles in /Applications/MNE-Python:"
            ls -d /Applications/MNE-Python/*.app || exit 1
            test `ls -d /Applications/MNE-Python/*.app | wc -l` -eq 5 || exit 1
            echo "Checking that the custom icon was set on the MNE folder in /Applications/MNE-Python"
            test -f /Applications/MNE-Python/Icon$'\r' || exit 1
          elif [[ "${{ runner.os }}" == "Linux" ]]; then
            echo "Checking that menu shortcuts were created …"
            pushd ~/.local/share/applications
            ls -l || exit 1
            echo "Checking for existence of .desktop files:"
            ls MNE-Python*.desktop || exit 1
            test `ls MNE-Python*.desktop | wc -l` -eq 5 || exit 1
            echo ""

            # … and patched to work around a bug in menuinst
            echo "Checking that incorrect Terminal entries have been removed"
            test `grep "Terminal=True"  MNE-Python*.desktop | wc -l` -eq 0 || exit 1
            test `grep "Terminal=False" MNE-Python*.desktop | wc -l` -eq 0 || exit 1
            echo ""

            echo "Checking that Terminal entries are correct…"
            test `grep "Terminal=true"  MNE-Python*.desktop | wc -l` -ge 1 || exit 1
            test `grep "Terminal=false" MNE-Python*.desktop | wc -l` -ge 1 || exit 1
            # Display their contents
            for f in MNE-Python*.desktop; do echo "📂 $f:"; cat "$f"; echo; done
            popd
          fi

          echo "Checking whether (Py)Qt is working"
          LD_DEBUG=libs python -c "from PyQt5.QtWidgets import QApplication, QWidget; app = QApplication([])"

          echo "Checking the deployed environment variables were set correctly upon environment activation"
          mamba env config vars list
          if [[ "${{ runner.os }}" == "macOS" ]]; then
            python -c "import os; x = os.getenv('CONDA_SUBDIR'); assert x == 'osx-64', f'CONDA_SUBDIR ({repr(x)}) != osx-64'" || exit 1
          fi
          # TODO: broken on Windows!
          if [[ "${{ runner.os }}" != "Windows" ]]; then
            python -c "import os; x = os.getenv('PYTHONNOUSERSITE'); assert x == '1', f'PYTHONNOUSERSITE ({repr(x)}) != 1'" || exit 1
            python -c "import os; x = os.getenv('MAMBA_NO_BANNER'); assert x == '1', f'MAMBA_NO_BANNER ({repr(x)}) != 1'" || exit 1
          fi

          echo "Running MNE's sys_info"
          mne sys_info

          echo "Trying to import MNE and all additional packages included in the installer"
          python -u tests/test_imports.py
          python -u tests/test_gui.py
          python -u tests/test_notebook.py
          python -u tests/test_json_versions.py

  # Release
  release:
    needs: [build, sign, test]
    if: github.ref_type == 'tag'
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}
    steps:
      # These names should correspond to MNE_INSTALLER_ARTIFACT_ID in tools/extract_version.sh

      - name: Download Linux installer
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-Linux-x86_64

      - name: Download Linux json
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-Linux-x86_64-json

      - name: Download Windows installer
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-Windows-x86_64

      - name: Download Windows json
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-Windows-x86_64-json

      - name: Download macOS Intel installer
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-macOS-x86_64

      - name: Download macOS Intel json
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-macOS-x86_64-json

      - name: Download macOS M1 installer
        uses: actions/download-artifact@v3
        with:
          name: MNE-Python-macOS-arm64

      # We don't have a JSON, as this is only produced upon successful
      # installation. Since we don't have an Apple Silicon runner on GHA,
      # disable this for now.
      #
      # - name: Download macOS M1 json
      #   uses: actions/download-artifact@v3
      #   with:
      #     name: MNE-Python-macOS-arm64-json

      - name: Check contents
        run: |
          ls -al ./

      - name: Release
        uses: ncipollo/release-action@v1
        with:
          artifacts: "MNE-Python-*.*"
          token: ${{ secrets.GITHUB_TOKEN }}
          draft: true
          prerelease: true
          allowUpdates: true
          generateReleaseNotes: true

#   add_release_hashes:
#     if: github.ref_type == 'tag'
#     needs: [build, test]
#     runs-on: ubuntu-latest

#     steps:
#       - name: Generate hashes of release artifacts
#         uses: MCJack123/ghaction-generate-release-hashes@v1
#         with:
#           hash-type: sha256
#           file-name: hashes.txt

#       - name: Add hashes to release
#         uses: ncipollo/release-action@v1
#         with:
#           artifacts: hashes.txt
#           token: ${{ secrets.GITHUB_TOKEN }}
#           draft: true
#           prerelease: true
#           allowUpdates: true
#           generateReleaseNotes: true