integration.yaml

name: Standard Library Integration Tests
on:
  pull_request:
    branches: [main]
    paths-ignore:  # ignore docs as they are built with Netlify.
      - '**/*.md'
      - 'site/**'
      - 'netlify.toml'
  push:
    branches: [main]
    paths-ignore:  # ignore docs as they are built with Netlify.
      - '**/*.md'
      - 'site/**'
      - 'netlify.toml'

defaults:
  run:  # use bash for all operating systems unless overridden
    shell: bash

env:  # Update this prior to requiring a higher minor version in go.mod
  GO_VERSION: "1.21"  # 1.xx == latest patch of 1.xx
  TINYGO_VERSION: "0.30.0"
  ZIG_VERSION: "0.11.0"

concurrency:
  # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run
  group: ${{ github.ref }}-${{ github.workflow }}-${{ github.actor }}
  cancel-in-progress: true

jobs:
  # This builds a zig test binary only if the same version tag hasn't been build before.
  # This saves time as we rarely update the zig version.
  build_zig_test_binary:
    name: Build Zig test binary
    runs-on: ubuntu-22.04
    env:
      ZIG_INSTALL: ~/zig-install
      ZIG_SOURCE: ~/zig-source

    steps:
      - name: Checkout wazero
        uses: actions/checkout@v3

      - uses: actions/cache@v3
        id: binary-cache
        with:
          # Use share the cache containing archives across OSes.
          enableCrossOsArchive: true
          key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }}
          path:
            ./zigbin/

      - name: Install Zig build
        if: steps.binary-cache.outputs.cache-hit != 'true'
        run: |
          mkdir -p ${{ env.ZIG_INSTALL }}
          curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-linux-x86_64-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_INSTALL }}

      - name: Download Zig source code
        if: steps.binary-cache.outputs.cache-hit != 'true'
        run: |
          mkdir -p ${{ env.ZIG_SOURCE }}
          curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_SOURCE }}

      - name: Build Stdlib test binary
        if: steps.binary-cache.outputs.cache-hit != 'true'
        # --test-no-exec allows building of the test Wasm binary without executing command.
        # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm
        run: |
          mkdir ${{ github.workspace }}/zigbin
          cd ${{ env.ZIG_SOURCE }}
          ${{ env.ZIG_INSTALL }}/zig test --test-no-exec -target wasm32-wasi --zig-lib-dir ./lib ./lib/std/std.zig
          _ZIG_TEST_BINARY_PATH=$(find . -name test.wasm)
          cp ${_ZIG_TEST_BINARY_PATH} ${{ github.workspace }}/zigbin/test.wasm

  zig:
    needs: build_zig_test_binary
    name: Zig (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2022]

    steps:
      - name: Checkout wazero
        uses: actions/checkout@v3

      - uses: actions/cache@v3
        id: binary-cache
        with:
          # Use share the cache containing archives across OSes.
          enableCrossOsArchive: true
          # We need this cache to run tests.
          fail-on-cache-miss: true
          key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }}
          path:
            ./zigbin/

      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Install wazero
        run: go install ./cmd/wazero

      # This runs the previously compiled Zig tests with wazero. If you need
      # to troubleshoot one, you can add "-hostlogging=filesystem" after
      # adding filter argument to the "Build Stdlib test binary" step.
      # e.g. --test-filter "Dir.Iterator but dir is deleted during iteration"
      - name: Run the test binary with wazero CLI
        run: wazero run -mount=:/ ./zigbin/test.wasm

  build_tinygo_test_binary:
    name: Build TinyGo test binary
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout wazero
        uses: actions/checkout@v3

      - uses: actions/cache@v3
        id: binary-cache
        with:
          # Use share the cache containing archives across OSes.
          enableCrossOsArchive: true
          key: tinygo-test-binaries-${{ env.TINYGO_VERSION }}
          path:
            ./tinygobin/

      - name: Install TinyGo
        if: steps.binary-cache.outputs.cache-hit != 'true'
        run: |  # installing via curl so commands are similar on OS/x
          tinygo_version=${{ env.TINYGO_VERSION }}
          curl -sSL https://github.com/tinygo-org/tinygo/releases/download/v${tinygo_version}/tinygo${tinygo_version}.linux-amd64.tar.gz | sudo tar -C /usr/local -xzf -
          echo "TINYGOROOT=/usr/local/tinygo" >> $GITHUB_ENV
          echo "/usr/local/tinygo/bin" >> $GITHUB_PATH

      - uses: actions/setup-go@v4
        if: steps.binary-cache.outputs.cache-hit != 'true'
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Build Test Binaries
        if: steps.binary-cache.outputs.cache-hit != 'true'
        # The following list of packages is derived from:
        # https://github.com/tinygo-org/tinygo/blob/v0.28.1/Makefile#L281-L322
        # Note:
        #  - index/suffixarray is extremely slow, so skip it.
        #  - compress/zlib is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/compress/zlib/writer_test.go#L16-L30
        #  - debug/macho is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/debug/macho/file_test.go#L25
        run: |
          mkdir ./tinygobin
          for value in container/heap \
            container/list \
            container/ring \
            crypto/des \
            crypto/md5 \
            crypto/rc4 \
            crypto/sha1 \
            crypto/sha256 \
            crypto/sha512 \
            embed/internal/embedtest \
            encoding \
            encoding/ascii85 \
            encoding/base32 \
            encoding/csv \
            encoding/hex \
            go/scanner \
            hash \
            hash/adler32 \
            hash/crc64 \
            hash/fnv \
            html \
            internal/itoa \
            internal/profile \
            math \
            math/cmplx \
            net \
            net/http/internal/ascii \
            net/mail \
            os \
            path \
            reflect \
            sync \
            testing \
            testing/iotest \
            text/scanner \
            unicode \
            unicode/utf16 \
            unicode/utf8
          do
            tinygo test -target wasi -c -o ./tinygobin/${value/\//_}.test $value
          done

  tinygo:
    needs: build_tinygo_test_binary
    name: TinyGo (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2022]

    steps:
      - name: Checkout wazero
        uses: actions/checkout@v3

      - uses: actions/cache@v3
        id: binary-cache
        with:
          # Use share the cache containing archives across OSes.
          enableCrossOsArchive: true
          # We need this cache to run tests.
          fail-on-cache-miss: true
          key: tinygo-test-binaries-${{ env.TINYGO_VERSION }}
          path:
            ./tinygobin/

      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Install wazero
        run: go install ./cmd/wazero

      # This runs the previously compiled TinyGo tests with wazero. If you need
      # to troubleshoot one, you can add "-hostlogging=filesystem" and also a
      # trailing argument narrowing which test to execute.
      # e.g. "-test.run '^TestStatBadDir$'"
      - name: Run standard library tests
        run: |
          cd ./tinygobin
          for bin in *.test; do
            echo $bin
            wazero run -mount=:/ -mount=:/tmp $bin -- -test.v
          done

  wasi-testsuite:
    name: wasi-testsuite
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2022]

    steps:
      - uses: actions/cache@v3
        id: cache
        with:
          path:
            ~/go/pkg/mod
          key: integration-test-wasi-testsuite-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}

      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Checkout wazero
        uses: actions/checkout@v3

      - name: Install wazero
        run: go install ./cmd/wazero

      - name: Checkout wasi-testsuite
        uses: actions/checkout@v3
        with:
          repository: WebAssembly/wasi-testsuite
          # prod/testsuite-base branch, as of May 12, 2023.
          # TODO: once the wasi-testsuite is stable, we should use the latest tag instead of a branch.
          ref: c9c751586fd86b321d595bbef13f2c7403cfdbc5
          path: wasi-testsuite

      - name: Initialize Python environment
        uses: actions/setup-python@v4
        with:
          python-version: '3.11' # latest version of python 3
          cache: pip

      - name: Install dependencies
        working-directory: wasi-testsuite/test-runner
        run: |
          python3 -m pip install -r requirements.txt

      - name: Run all wasi-testsuite
        working-directory: wasi-testsuite
        run: |
          python3 test-runner/wasi_test_runner.py \
            -t ./tests/assemblyscript/testsuite/ \
            ./tests/c/testsuite/ \
            ./tests/rust/testsuite/ \
            -f ../.github/wasi_testsuite_skip.json \
            -r ../.github/wasi_testsuite_adapter.py

  gojs_stdlib:
    name: Go (js) (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
      matrix:
        os: [ubuntu-22.04, macos-12] # GOOS=js isn't supposed to work on windows. See #1222

    steps:
      - uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Checkout wazero
        uses: actions/checkout@v3

      - name: Install wazero
        run: go install ./cmd/wazero

      - name: Build gojs test binaries
        env:
          GOOS: js
          GOARCH: wasm
        run: | # Only test os package as this is being replaced by GOOS=wasip1
          mkdir ~/bin && cd ~/bin
          go test -c -o os.wasm os

      - name: Run tests
        run: |  # skip tests that use functionality not also used in GOOS=wasip1
          cd $(go env GOROOT)/src/os; wazero run -mount=/:/ ~/bin/os.wasm -test.v -test.skip '^Test(Chmod|Truncate|LongPath|Chown|FileChown).*$'

  go_tests:
    # Due to the embedding of the GOROOT of the building env(https://github.com/golang/go/blob/3c59639b902fada0a2e5a6a35bafd10fc9183b89/src/os/os_test.go#L112),
    # we have to build and cache on each OS unlike others in this file.
    name: Go (${{ matrix.os }}, Go-${{ matrix.go-version }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false # don't fail fast as sometimes failures are arch/OS specific
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2022]
        go-version:
          - "1.21"  # Current Go version && The only version that supports wasip1.

    steps:
      - id: setup-go
        uses: actions/setup-go@v4
        with:
          go-version: ${{ matrix.go-version }}

      - name: Cache Go test binaries
        id: cache-go-test-binaries
        uses: actions/cache@v3
        with:
          path: ~/tests
          # Use precise Go version from setup-go as patch version differences can effect tests.
          key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }}

      - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }}
        name: Build Test Binaries
        run: |
          mkdir ~/tests
          cd $(go env GOROOT)
          # Choose important packages to limit execution time.
          for value in src/archive/tar \
            src/bufio \
            src/bytes \
            src/context \
            src/encoding/ascii85 \
            src/encoding/asn1 \
            src/encoding/base32 \
            src/encoding/base64 \
            src/encoding/binary \
            src/encoding/csv \
            src/encoding/gob \
            src/encoding/hex \
            src/encoding/json \
            src/encoding/pem \
            src/encoding/xml \
            src/errors \
            src/expvar \
            src/flag \
            src/fmt \
            src/hash \
            src/hash/adler32 \
            src/hash/crc32 \
            src/hash/crc64 \
            src/hash/fnv \
            src/hash/maphash \
            src/io \
            src/io/fs \
            src/io/ioutil \
            src/log \
            src/log/syslog \
            src/maps \
            src/math \
            src/math/big \
            src/math/bits \
            src/math/cmplx \
            src/math/rand \
            src/mime \
            src/mime/multipart \
            src/mime/quotedprintable \
            src/os \
            src/os/exec \
            src/os/signal \
            src/os/user \
            src/path \
            src/reflect \
            src/regexp \
            src/regexp/syntax \
            src/runtime \
            src/runtime/internal/atomic \
            src/runtime/internal/math \
            src/runtime/internal/sys \
            src/slices \
            src/sort \
            src/strconv \
            src/strings \
            src/sync \
            src/sync/atomic \
            src/syscall \
            src/testing \
            src/testing/fstest \
            src/testing/iotest \
            src/testing/quick \
            src/time
          do
            echo "GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value"
            GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value
          done

      - name: Checkout wazero
        uses: actions/checkout@v3

      - name: Install wazero
        run: go install ./cmd/wazero

      - if: ${{ runner.os != 'Windows' }}
        name: Run standard library tests
        run: |
          echo "Running $(find ~/tests -name *.test | wc -l) test binaries"

          # Go tests often look for files relative to the source. Change to the corresponding directory.
          for bin in ~/tests/*.test; do
            dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/}
            pushd $(go env GOROOT)/$dir
            wazero run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v
            popd
          done

      - if: ${{ runner.os == 'Windows' }}
        name: Run standard library tests
        # Ack failures on Windows. https://github.com/tetratelabs/wazero/issues/1410
        continue-on-error: true
        run: |
          GOOS=$(go env GOOS)
          echo "Running $(find ~/tests -name *.test | wc -l) test binaries"

          MOUNT=c:\\:/
          SCRIPT="$HOME/tests.cmd"
          # Trim `/c` from the in-Wasm GOROOT.
          REAL_GOROOT=$(go env GOROOT)
          GOROOT=$(cygpath -u $REAL_GOROOT | cut -c 3-)
          # Append early exit on cmd.
          POSTFIX="if %errorlevel% neq 0 exit /b %errorlevel%"
          RUNNER="cmd //c %USERPROFILE%\tests.cmd"
          EXTRAPARAMS="-mount=%TEMP%:/tmp"

          # Go tests often look for files relative to the source. Change to the corresponding directory.
          for bin in ~/tests/*.test; do
            dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/}
            pushd $REAL_GOROOT/$dir

            # Trim `/c` from the in-Wasm pwd.
            IN_WASM_PWD=$(pwd | cut -c 3-)
            # Convert to a Windows path.
            bin=`cygpath -w $bin`

            # Create a script with all the tests (do not run yet).
            echo ${MOUNT} ${IN_WASM_PWD} $GOROOT/$dir
            COMMAND="wazero run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v"
            echo $COMMAND >> $SCRIPT
            # Uncomment the following line for early exit on error on Windows.
            # Otherwise the tests will report are successful evne on failure.
            # echo $POSTFIX >> $SCRIPT
            popd
          done

          # Run all the tests in their own shell.
          $RUNNER