test-server-all.yml

name: Build and Run Tests on CC, MC, and BTS Containers

on:
  pull_request:
  push:
    branches:
      - main
      - "release/**"

env:
  DOCKER_BUILDKIT: 1
  COMPOSE_DOCKER_CLI_BUILD: 1
  COMPOSE_FILES_OPT: "-f ./docker-compose.medium.yml -f ./docker-compose.core.yml -f ./docker-compose.benchmark.yml -f ./docker-compose.small.yml -f ./docker-compose.datadog.yml -f ./docker-compose.dev.yml"

jobs:
  lint:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Install clang-format tool
        run: |
          wget https://apt.llvm.org/llvm.sh
          chmod +x llvm.sh
          sudo ./llvm.sh 14
          sudo apt-get update
          sudo apt-get install clang-format-14
      - name: Check if *.{cpp|hpp} files are formatted
        run: |
          cd ${{ github.workspace }}/packages/server/
          make check

  small_test_cc:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Build cc with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "small_cc"
          cache-key: ${{ hashFiles('./packages/server/computation_container/**') }}
          dockerfile-path: ./packages/server/computation_container/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "small_tmp"

      - name: Run bazel test in computation_container
        run: make test t=./computation_container m=run p=small
        working-directory: ./scripts

  small_test_mc:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Build mc with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "small_mc"
          cache-key: ${{ hashFiles('./packages/server/manage_container/**') }}
          dockerfile-path: ./packages/server/manage_container/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "small"

      - name: Run Go test in manage_container
        run: make test t=./manage_container m=run p=small
        working-directory: ./scripts

  small_test_bts:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Build bts with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "small_bts"
          cache-key: ${{ hashFiles('./packages/server/beaver_triple_service/**') }}
          dockerfile-path: ./packages/server/beaver_triple_service/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "small"

      - name: Run test in beaver_triple_service
        run: make test t=./beaver_triple_service m=run
        working-directory: ./scripts

  small_test:
    runs-on: ubuntu-latest
    needs: [small_test_cc, small_test_mc, small_test_bts]

    steps:
      - name: Merge All small_test
        run: echo "ok"

  medium_test_cc:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Log into registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Build cc with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "medium_cc"
          cache-key: ${{ hashFiles('./packages/server/computation_container/**') }}
          dockerfile-path: ./packages/server/computation_container/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "medium"

      - name: Build bts with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "dev_bts"
          cache-key: ${{ hashFiles('./packages/server/beaver_triple_service/**') }}
          dockerfile-path: ./packages/server/beaver_triple_service/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "dev"

      - name: Run bazel test in computation_container
        run: make test t=./computation_container m=run p=medium
        working-directory: ./scripts

  medium_test_mc:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Build mc with cache
        uses: ./.github/actions/build_images_with_cache
        with:
          container-name: "medium_mc"
          cache-key: ${{ hashFiles('./packages/server/manage_container/**') }}
          dockerfile-path: ./packages/server/manage_container/Dockerfile
          builder-name: ${{ steps.buildx.outputs.name }}
          target: "medium"

      - name: Run bazel test in manage_container
        run: make test t=./manage_container m=run p=medium
        working-directory: ./scripts

  medium_test_libclient:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test_name:
          [
            correl,
            get_computation_result,
            get_elapsed_time,
            job_error_info,
            join,
            mean,
            parallel,
            send_share,
            string,
            sum,
            token,
            variance,
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
          submodules: true

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Log into registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin

      - uses: ./.github/actions/prepare_libclient_test

      - name: Run container test
        run: docker-compose ${{ env.COMPOSE_FILES_OPT }} run medium-libc pytest src/tests/test_${{ matrix.test_name }}.py -s -v -log-cli-level=DEBUG
        working-directory: ./scripts

  medium_test_container_up:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
          submodules: true

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Log into registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin

      - uses: ./.github/actions/prepare_container_test

      - name: Run container test
        run: |
          pipenv sync
          pipenv install --skip-lock ../../packages/client/libclient-py
          pipenv run pytest ./tests/test_up.py -s -v -log-cli-level=DEBUG
        working-directory: ./scripts/container_test

  medium_test_container_restart:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test_name:
          [
            send_share,
            execute_computation,
            get_computation_result,
            execute_multiple
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
          submodules: true

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Log into registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin

      - uses: ./.github/actions/prepare_container_test

      - name: Run container test
        run: |
          pipenv sync
          pipenv install --skip-lock ../../packages/client/libclient-py
          pipenv run pytest ./tests/test_restart_request.py::test_success_${{ matrix.test_name }}_with_restart -s -v -log-cli-level=DEBUG
        working-directory: ./scripts/container_test

  medium_test_container_down:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test_name:
          [
            send_share,
            execute_computation,
            get_computation_result,
          ]
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
          submodules: true

      - name: Update .bazelrc for using remote cache
        run: |
          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
        env:
          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

      - name: Log into registry
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin

      - uses: ./.github/actions/prepare_container_test

      - name: Run container test
        run: |
          pipenv sync
          pipenv install --skip-lock ../../packages/client/libclient-py
          pipenv run pytest ./tests/test_down_request.py::test_failed_${{ matrix.test_name }}_with_down -s -v -log-cli-level=DEBUG
        working-directory: ./scripts/container_test

  medium_test_container:
    runs-on: ubuntu-latest
    needs:
      [
        medium_test_container_up,
        medium_test_container_restart,
        medium_test_container_down
      ]
    steps:
      - name: Merge All medium_test_container
        run: echo "ok"

  medium_test:
    runs-on: ubuntu-latest
    needs:
      [
        medium_test_cc,
        medium_test_mc,
        medium_test_libclient,
        medium_test_container
      ]
    steps:
      - name: Merge All medium_test
        run: echo "ok"