main.yml

name: CI

on:
  pull_request:

  push:
    branches: [ master ]

jobs:
  build-linux-nix:
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Install Nix
      uses: cachix/install-nix-action@v27

    - uses: cachix/cachix-action@v15
      with:
        name: futhark
        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'

    - name: Build Futhark
      run: nix-build --argstr suffix nightly-linux-x86_64 --argstr commit $GITHUB_SHA

    - uses: actions/upload-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz
        path: result/futhark-nightly-linux-x86_64.tar.xz

  build-linux-cabal:
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Workaround runner image issue
      # https://github.com/actions/runner-images/issues/7061
      run: sudo chown -R $USER /usr/local/.ghcup

    - uses: haskell-actions/setup@v2
      with:
        ghc-version: '9.8.1'

    - uses: actions/cache@v4
      name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle
      with:
        path: |
          ~/.cabal/packages
          ~/.cabal/store
          dist-newstyle
        key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }}

    - name: cabal check
      run: cabal check

    - name: Build Futhark
      run: |
        cabal --version
        ghc --version
        make configure
        make build
        make install

  build-mac-cabal:
    runs-on: macos-latest

    steps:
    - uses: actions/checkout@v4

    - name: Install dependencies
      run: |
        brew install cabal-install ghc sphinx-doc
        echo "/opt/homebrew/opt/sphinx-doc/bin" >> $GITHUB_PATH
        echo "/opt/homebrew/opt/ghc/bin" >> $GITHUB_PATH

    - uses: actions/cache@v4
      name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle
      with:
        path: |
          ~/.cabal/packages
          ~/.cabal/store
          dist-newstyle
        key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }}

    - name: Build
      run: |
        cp -r tools/release/skeleton futhark-nightly-macos-x86_64

        mkdir -p futhark-nightly-macos-x86_64/bin
        cabal v2-update
        cabal install --install-method=copy --overwrite-policy=always --installdir=futhark-nightly-macos-x86_64/bin
        mkdir -p futhark-nightly-macos-x86_64/share/man/man1/
        (cd docs; make man)
        cp -r docs/_build/man/* futhark-nightly-macos-x86_64/share/man/man1/

        mkdir -p futhark-nightly-macos-x86_64/share/futhark
        cp LICENSE futhark-nightly-macos-x86_64/share/futhark/
        echo "${GITHUB_SHA}" > futhark-nightly-macos-x86_64/commit-id
        tar -Jcf futhark-nightly-macos-x86_64.tar.xz futhark-nightly-macos-x86_64

    - uses: actions/upload-artifact@v4
      with:
        name: futhark-nightly-macos-x86_64.tar.xz
        path: futhark-nightly-macos-x86_64.tar.xz

  build-windows-cabal:
    runs-on: windows-latest

    steps:
    - uses: actions/checkout@v4

    - id: setup-haskell
      uses: haskell-actions/setup@v2
      with:
        ghc-version: '9.8.1'

    - uses: actions/cache@v4
      name: Cache cabal stuff
      with:
        path: |
          ${{ steps.setup-haskell.outputs.cabal-store }}
          dist-newstyle
        key: ${{ runner.os }}-cabal-${{ hashFiles('futhark.cabal', 'cabal.project') }}

    - name: Build
      shell: bash
      run: |
        cabal update
        try() { cabal install --install-method=copy --overwrite-policy=always --installdir=.; }
        try || try || try
    - uses: vimtor/action-zip@v1.2
      with:
        files: futhark.exe
        dest: futhark-nightly-windows-x86_64.zip

    - uses: actions/upload-artifact@v4
      with:
        name: futhark-nightly-windows-x86_64.zip
        path: futhark-nightly-windows-x86_64.zip


  haskell-test-style:
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Install Nix
      uses: cachix/install-nix-action@v27

    - uses: actions/cache@v4
      name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle
      with:
        path: |
          ~/.cabal/packages
          ~/.cabal/store
          dist-newstyle
        key: ${{ runner.os }}-${{ hashFiles('nix/sources.json') }}-style

    - name: Style check
      run: nix-shell --pure --run "make check"


  python-test-style:
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4

    - name: Install Nix
      uses: cachix/install-nix-action@v27

    - name: Style check
      run: nix-shell --pure --run "black --check ."

    - name: Type check
      run: nix-shell --pure --run "mypy ."

  # Fails mysteriously right now.
  #
  # build-docs:
  #   runs-on: ubuntu-22.04

  #   steps:
  #   - uses: actions/checkout@v4

  #   - name: Install Nix
  #     uses: cachix/install-nix-action@v18

  #   - uses: actions/cache@v4
  #     name: Cache ~/.cabal/packages, ~/.cabal/store and dist-newstyle
  #     with:
  #       path: |
  #         ~/.cabal/packages
  #         ~/.cabal/store
  #         dist-newstyle
  #       key: ${{ runner.os }}-${{ hashFiles('nix/sources.json', 'futhark.cabal', 'cabal.project') }}-haddock

  #   - name: Run haddock
  #     run: |
  #         nix-shell --pure --run "make configure"
  #         nix-shell --pure --run "make docs"

  test-interpreter:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        futhark test -i tests

  test-structure:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        futhark test -s tests

  test-c:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y python3-jsonschema
    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        export CFLAGS="-fsanitize=undefined -fsanitize=address -fno-sanitize-recover -O"
        futhark test -c --backend=c tests --no-tuning
        make -C tests_lib/c -j

  test-multicore:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y python3-jsonschema
    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        export CFLAGS="-fsanitize=undefined -fno-sanitize-recover -O"
        futhark test -c --backend=multicore tests --no-tuning
        make -C tests_lib/c FUTHARK_BACKEND=multicore

  test-ispc:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y python3-jsonschema

    - name: Install Nix
      uses: cachix/install-nix-action@v27

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - run: |
        export CFLAGS="-fsanitize=undefined -fno-sanitize-recover -O"
        nix-shell --run 'futhark test -c --backend=ispc tests --no-tuning'
        nix-shell --run 'make -C tests_lib/c FUTHARK_BACKEND=ispc'

  test-python:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y python3-numpy

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - run: |
        futhark test -c --no-terminal --no-tuning --backend=python --exclude=no_python --exclude=compiled tests
        make -C tests_lib/python -j

  test-oclgrind:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]
    steps:
    - uses: actions/checkout@v4
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y opencl-headers oclgrind nvidia-opencl-dev
    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz
    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        futhark test tests -c --no-terminal --backend=opencl --exclude=compiled --exclude=no_oclgrind --cache-extension=cache --pass-option=--build-option=-O0 --runner=tools/oclgrindrunner.sh

  test-pyoclgrind:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]
    steps:
    - uses: actions/checkout@v4
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y opencl-headers oclgrind
    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz
    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - run: |
        set -e
        python -m venv virtualenv
        source virtualenv/bin/activate
        pip install 'numpy<2.0.0' pyopencl jsonschema
        futhark test tests -c --no-terminal --backend=pyopencl --exclude=compiled --exclude=no_oclgrind --cache-extension=cache --pass-option=--build-option=-O0 --runner=tools/oclgrindrunner.sh

  test-opencl:
    runs-on: hendrix
    needs: [build-linux-nix]
    if: github.repository == 'diku-dk/futhark'
    steps:
    - uses: actions/checkout@v4

    - uses: ./.github/actions/futhark-slurm
      with:
        script: |
          set -e
          python -m venv virtualenv
          source virtualenv/bin/activate
          pip install jsonschema
          module unload cuda
          module load cuda/11.8
          futhark test tests \
            --backend=opencl \
            --cache-extension=cache
          FUTHARK_BACKEND=opencl make -C tests_lib/c -j
        slurm-options: -p gpu --time=0-01:00:00 --gres=gpu:1 --job-name=fut-opencl-test --exclude=hendrixgpu05fl,hendrixgpu06fl

  test-pyopencl:
    runs-on: hendrix
    needs: [build-linux-nix]
    if: github.repository == 'diku-dk/futhark'
    steps:
    - uses: actions/checkout@v4

    - uses: ./.github/actions/futhark-slurm
      with:
        script: |
          set -e
          python -m venv virtualenv
          source virtualenv/bin/activate
          pip install numpy pyopencl jsonschema
          module unload cuda
          module load cuda/11.8
          futhark test tests --no-terminal --backend=pyopencl
        slurm-options: -p gpu --time=0-02:00:00 --gres=gpu:1 --job-name=fut-pyopencl-test --exclude=hendrixgpu05fl,hendrixgpu06fl

  test-cuda:
    runs-on: cuda
    needs: [build-linux-nix]
    if: github.repository == 'diku-dk/futhark'
    steps:
    - uses: actions/checkout@v4

    - uses: ./.github/actions/futhark-slurm
      with:
        script: |
          set -e
          python -m venv virtualenv
          source virtualenv/bin/activate
          pip install jsonschema
          module unload cuda
          module load cuda/11.8
          nvidia-smi --query-gpu=gpu_name --format=csv,noheader
          futhark test tests \
            --backend=cuda \
            --cache-extension=cache \
            --concurrency=8
          FUTHARK_BACKEND=cuda make -C tests_lib/c -j
        slurm-options: -p gpu --time=0-01:00:00 --gres=gpu:a100:1 --job-name=fut-cuda-test

  test-hip:
    runs-on: MI100
    needs: [build-linux-nix]
    if: github.repository == 'diku-dk/futhark'
    steps:
    - uses: actions/checkout@v4

    - run: |
        python -m venv virtualenv
        source virtualenv/bin/activate
        pip install jsonschema

    - uses: ./.github/actions/futhark-slurm
      with:
        script: |
          set -e
          futhark test tests --no-terminal --backend=hip --concurrency=8
          source virtualenv/bin/activate
          FUTHARK_BACKEND=hip make -C tests_lib/c -j

  test-wasm:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: mymindstorm/setup-emsdk@v12
      with:
        version: 2.0.18
        actions-cache-folder: 'emsdk-cache'

    - uses: actions/setup-node@v3.5.1
      with:
        node-version: '16.x'

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - name: Run tests
      run: |
        node --version
        export EMCFLAGS="-sINITIAL_MEMORY=2147418112 -O1" # 2gb - 64kb... largest value of memory
        futhark test \
          -c \
          --backend=wasm \
          --runner=./tools/node-simd.sh \
          --no-tuning \
          --exclude=no_wasm tests

  test-wasm-multicore:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: mymindstorm/setup-emsdk@v12
      with:
        version: 2.0.18
        actions-cache-folder: 'emsdk-mc-cache'

    - uses: actions/setup-node@v3.5.1
      with:
        node-version: '16.x'

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - name: Run tests
      run: |
        node --version
        export EMCFLAGS="-sINITIAL_MEMORY=2147418112 -O1 -s PTHREAD_POOL_SIZE=12" # 2gb - 64kb... largest value of memory
        futhark test -c --backend=wasm-multicore --runner=./tools/node-threaded.sh --no-tuning --exclude=no_wasm tests

  test-wasm-lib:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: mymindstorm/setup-emsdk@v12
      with:
        version: 2.0.18
        actions-cache-folder: 'emsdk-cache'

    - uses: actions/setup-node@v3.5.1
      with:
        node-version: '16.x'

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - name: Run tests
      run: |
        make -C tests_lib/javascript

  test-ir-parser:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - run: |
        tools/testparser.sh tests

  test-formatter:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - run: |
        tools/testfmt.sh tests

  test-tools:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix]

    steps:
    - name: Install OS dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y ffmpeg oclgrind

    - uses: actions/checkout@v4

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - name: Install from nightly tarball
      run: |
        tar xvf futhark-nightly-linux-x86_64.tar.xz
        make -C futhark-nightly-linux-x86_64/ install PREFIX=$HOME/.local
        echo "$HOME/.local/bin" >> $GITHUB_PATH

    - run: |
        cd tests_pkg && sh test.sh

    - run: |
        cd tests_literate && sh test.sh

    - run: |
        cd tests_repl && sh test.sh

    - run: |
        cd tests_bench && sh test.sh

    - run: |
        cd tests_adhoc && sh test.sh

    - run: |
        cd tests_fmt && sh test.sh

    - run: |
        futhark doc -o prelude-docs /dev/null
        tar -Jcf prelude-docs.tar.xz prelude-docs

    - uses: actions/upload-artifact@v4
      with:
        name: prelude-docs.tar.xz
        path: prelude-docs.tar.xz

  deploy-nightly:
    runs-on: ubuntu-22.04
    needs: [build-linux-nix, build-mac-cabal, build-windows-cabal, test-tools]
    if: github.ref == 'refs/heads/master'

    steps:
    - name: Install SSH key
      uses: shimataro/ssh-key-action@v2
      with:
        key: ${{ secrets.SSHKEY }}
        known_hosts: ${{ secrets.KNOWN_HOSTS }}

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-windows-x86_64.zip

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-macos-x86_64.tar.xz

    - uses: actions/download-artifact@v4
      with:
        name: futhark-nightly-linux-x86_64.tar.xz

    - uses: actions/download-artifact@v4
      with:
        name: prelude-docs.tar.xz

    - name: scp tarballs to server
      run: |
        scp -o StrictHostKeyChecking=no futhark-nightly-*-x86_64.{tar.xz,zip} futhark@futhark-lang.org:/var/www/htdocs/futhark-lang.org/releases

    - name: copy docs to server
      run: |
        tar -xf prelude-docs.tar.xz
        rsync -rv -e 'ssh -o "StrictHostKeyChecking no"' prelude-docs/* futhark@futhark-lang.org:/var/www/htdocs/futhark-lang.org/docs/prelude/

    - name: make nightly release
      uses: "marvinpinto/action-automatic-releases@latest"
      with:
        repo_token: "${{ secrets.GITHUB_TOKEN }}"
        automatic_release_tag: "nightly"
        prerelease: true
        title: "nightly"
        files: |
          futhark-nightly-linux-x86_64.tar.xz
          futhark-nightly-macos-x86_64.tar.xz
          futhark-nightly-windows-x86_64.zip