publish-crossbuild.yml

# SPDX-License-Identifier: Apache-2.0
#
# Copyright (c) 2021 Patrick Dung

name: Release using cross build
on:
  push:
    branches:
      - main
    paths:
      - 'release-versions/*'
env:
  CARGO_TERM_COLOR: always
  # CONTAINER_REGISTRY: quay.io
  # CONTAINER_REPOSITORY: patrickdung/docker-images-meilisearch
  CONTAINER_REGISTRY: ghcr.io
  # ## GH Repository must be lower case when using in GH action
  CONTAINER_REPOSITORY: ${{ github.repository }}
  ORIGINAL_SOURCECODE_URL: https://github.com/meilisearch/MeiliSearch

jobs:
  init-env:
    name: Set env outputs
    runs-on: ubuntu-latest
    continue-on-error: true
    permissions:
      actions: none
      checks: none
      contents: none
      deployments: none
      issues: none
      packages: none
      pull-requests: none
      repository-projects: none
      security-events: none
      statuses: none
      # id-token: write # needed for signing the images with GitHub OIDC **not production ready**
    outputs:
      container_registry_base_uri: ${{ steps.set-env.outputs.container_registry_base_uri }}
    steps:
      - name: Set env for later jobs
        id: set-env
        run: |
          echo "container_registry_base_uri=$(echo $CONTAINER_REGISTRY/$(echo $CONTAINER_REPOSITORY | tr 'A-Z' 'a-z'))" >> $GITHUB_OUTPUT

  publish-with-crossbuild:
    name: Publish to Github
    needs: [init-env]
    runs-on: ${{ matrix.os }}
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - build: aarch64
            os: ubuntu-20.04
            target: aarch64-unknown-linux-gnu
            linker: gcc-aarch64-linux-gnu
            use-cross: true
            asset_name: meilisearch-linux-aarch64
          - build: linux
            os: ubuntu-20.04
            target: x86_64-unknown-linux-gnu
            use-cross: false
            asset_name: meilisearch-linux-x86_64
          - build: aarch64
            os: ubuntu-20.04
            target: aarch64-unknown-linux-musl
            linker: gcc-aarch64-linux-gnu
            use-cross: true
            asset_name: meilisearch-linux-aarch64-musl
          - build: linux
            os: ubuntu-20.04
            target: x86_64-unknown-linux-musl
            use-cross: true
            asset_name: meilisearch-linux-x86_64-musl
    outputs:
      remote_branch_name: ${{ steps.get-remote-branch-name.outputs.remote_branch_name }}

    steps:
      - name: "Get branch name of latest release version of Meili"
        id: get-remote-branch-name
        run: |
          curl -sL https://api.github.com/repos/meilisearch/MeiliSearch/releases | \
            jq -r ".[].tag_name" | grep -v rc | sort -r -V | head -n 1 > /tmp/meilisearch-latest-branch-name

          echo "REMOTE_BRANCH_NAME=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_ENV
          echo "remote_branch_name=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_OUTPUT

      - name: Checkout repository of official repo for compiling
        uses: actions/checkout@v2
        with:
          repository: meilisearch/MeiliSearch
          ref: ${{ env.REMOTE_BRANCH_NAME }}

      - name: Installing Rust toolchain
        uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
          profile: minimal
          target: ${{ matrix.target }}
          override: true

      - name: APT update
        run: |
          sudo apt update

      - name: Install target specific tools
        if: matrix.use-cross
        run: |
          sudo apt-get install -y ${{ matrix.linker }}

      - name: Configure target aarch64 GNU
        if: matrix.target == 'aarch64-unknown-linux-gnu'
        ## Environment variable is not passed using env:
        ## LD gold won't work with MUSL
        # env:
        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
        ## LD gold had problem with meili v0.30 aarch glibc build
        run: |
          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
          ## echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV

      - name: Configure target aarch64 MUSL
        if: matrix.target == 'aarch64-unknown-linux-musl'
        # env:
        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
        run: |
          sudo apt-get install -y musl-tools
          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV

      - name: Configure target x86_64 MUSL
        if: matrix.target == 'x86_64-unknown-linux-musl'
        run: |
          sudo apt-get install -y musl-tools

      - name: Cargo build
        uses: actions-rs/cargo@v1
        with:
          command: build
          use-cross: ${{ matrix.use-cross }}
          args: --release --target ${{ matrix.target }}

      # Strip debuginfo for target aarch64 GNU
      # MUSL binaries are static linked
      - name: Strip debuginfo for target aarch64 GNU
        if: matrix.target == 'aarch64-unknown-linux-gnu'
        run: |
          /usr/bin/aarch64-linux-gnu-strip --strip-debug --target=elf64-littleaarch64 target/${{ matrix.target }}/release/meilisearch -o target/${{ matrix.target }}/release/meilisearch-stripped

      - name: Strip debuginfo for target x86_64 GNU
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        run: |
          strip --strip-debug target/${{ matrix.target }}/release/meilisearch -o target/${{ matrix.target }}/release/meilisearch-stripped

      - name: Create checksum file for the binaries
        run: |
          cd target/${{ matrix.target }}/release
          sha256sum meilisearch | awk '{print $1, "${{matrix.asset_name}}"}' > ${{matrix.asset_name}}.sha256sum
          if [ -e meilisearch-stripped ]; then
             sha256sum meilisearch-stripped | awk '{print $1, "${{matrix.asset_name}}-stripped"}' > ${{matrix.asset_name}}-stripped.sha256sum
          fi

      - name: List target output files
        run: ls -lR ./target

      - name: Upload the binary to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.PUBLISH_TOKEN }}
          # repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: target/${{ matrix.target }}/release/meilisearch
          asset_name: ${{ matrix.asset_name }}
          tag: ${{ env.REMOTE_BRANCH_NAME }}
          overwrite: true

      - name: Upload stripped binary to release (aarch64/x86_64 GNU only)
        if: matrix.target == 'aarch64-unknown-linux-gnu' || matrix.target == 'x86_64-unknown-linux-gnu'
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.PUBLISH_TOKEN }}
          # repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: target/${{ matrix.target }}/release/meilisearch-stripped
          asset_name: ${{ matrix.asset_name }}-stripped
          tag: ${{ env.REMOTE_BRANCH_NAME }}
          overwrite: true

      - name: Upload checksum files to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.PUBLISH_TOKEN }}
          file: target/${{ matrix.target }}/release/*.sha256sum
          file_glob: true
          tag: ${{ env.REMOTE_BRANCH_NAME }}
          overwrite: true

  build-docker-image:
    needs: [init-env, publish-with-crossbuild]
    name: Build Docker Images
    runs-on: ubuntu-latest
    continue-on-error: true
    permissions:
      actions: none
      checks: none
      contents: read
      deployments: none
      issues: none
      packages: write
      pull-requests: none
      repository-projects: none
      # GH action/scanners for sarif reports
      security-events: write
      statuses: none
      # id-token: write # needed for signing the images with GitHub OIDC **not production ready**
    outputs:
      container_digest_amd64: ${{ steps.get-container-digest-amd64.outputs.container_digest }}
      container_digest_arm64: ${{ steps.get-container-digest-arm64.outputs.container_digest }}

    steps:
      - name: "Fetch branch name of latest version of Meili"
        run: |
          curl -sL https://api.github.com/repos/meilisearch/MeiliSearch/releases/latest | \
          jq -r ".tag_name" > /tmp/meilisearch-latest-branch-name
          echo "REMOTE_BRANCH_NAME=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_ENV

      - name: Checkout repository
        uses: actions/checkout@v2
        with:
          repository: patrickdung/MeiliSearch-crossbuild
          ref: main

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to container registry provider
        uses: docker/login-action@v2
        with:
          registry: ${{ env.CONTAINER_REGISTRY }}
          # GitLab
          # username: ${{ secrets.GITLAB_USERNAME }}
          # password: ${{ secrets.GITLAB_TOKEN }}
          # Quay.io
          # username: ${{ secrets.QUAY_USERNAME }}
          # password: ${{ secrets.QUAY_TOKEN }}
          # GitHub
          username: ${{ github.actor }}
          password: ${{ secrets.PUBLISH_TOKEN }}

      - name: Install Cosign GH action
        uses: sigstore/cosign-installer@main
        with:
          cosign-release: 'v1.13.1'

      - name: Build and push to container registry
        uses: docker/build-push-action@v4
        env:
          DOCKER_CONTENT_TRUST: 1
        with:
          context: .
          build-args: |
            MEILISEARCH_VERSION=${{ env.REMOTE_BRANCH_NAME }}
            SOURCE_BINARY_BASEURL=https://github.com/patrickdung/MeiliSearch-crossbuild/releases/download
            LABEL_IMAGE_URL=${{ env.ORIGINAL_SOURCECODE_URL }}
            LABEL_IMAGE_SOURCE=https://github.com/${{ github.repository }}
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            ${{ needs.init-env.outputs.container_registry_base_uri }}:${{ env.REMOTE_BRANCH_NAME }}

      - name: "Get the digest of container (amd64)"
        id: get-container-digest-amd64
        run: |
          skopeo inspect --raw docker://${{needs.init-env.outputs.container_registry_base_uri}}:${{env.REMOTE_BRANCH_NAME}} | \
            jq -r '.manifests[] | select(.platform .architecture=="amd64" and .platform .os=="linux") | .digest' > /tmp/container-digest-amd64
          echo "container_digest=$(cat /tmp/container-digest-amd64)" >> $GITHUB_OUTPUT
      - name: "Get the digest of container (arm64)"
        id: get-container-digest-arm64
        run: |
          skopeo inspect --raw docker://${{needs.init-env.outputs.container_registry_base_uri}}:${{env.REMOTE_BRANCH_NAME}} | \
            jq -r '.manifests[] | select(.platform .architecture=="arm64" and .platform .os=="linux") | .digest' > /tmp/container-digest-arm64
          echo "container_digest=$(cat /tmp/container-digest-arm64)" >> $GITHUB_OUTPUT

      - name: Use Cosign to sign the image recursively
        run: |
          echo -n "${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }}" | \
            cosign sign --recursive --key <(echo -n "${{ secrets.COSIGN_PRIVATE_KEY }}") \
            "${{ needs.init-env.outputs.container_registry_base_uri }}:${{ env.REMOTE_BRANCH_NAME }}"

  build-sbom-and-scanning:
    needs: [init-env, publish-with-crossbuild, build-docker-image]
    name: Build SBOM and image scanning
    runs-on: ubuntu-latest
    continue-on-error: true
    permissions:
      actions: none
      checks: none
      contents: read
      deployments: none
      issues: none
      packages: write
      pull-requests: none
      repository-projects: none
      # GH action/scanners for sarif reports
      security-events: write
      statuses: none
      # id-token: write # needed for signing the images with GitHub OIDC **not production ready**
    strategy:
      # Anchore action produces the same filename for sarif on different platforms
      max-parallel: 1
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            platform_image_uri: "${{needs.init-env.outputs.container_registry_base_uri}}@${{ needs.build-docker-image.outputs.container_digest_amd64 }}"
          - arch: arm64
            platform_image_uri: "${{needs.init-env.outputs.container_registry_base_uri}}@${{ needs.build-docker-image.outputs.container_digest_arm64 }}"

    steps:
      - name: Set env REMOTE_BRANCH_NAME
        run: |
          echo "REMOTE_BRANCH_NAME=${{ needs.publish-with-crossbuild.outputs.remote_branch_name }}" >> $GITHUB_ENV

      - name: Checkout this repository
        # some vuln scanner want to have the Dockerfile
        uses: actions/checkout@v2
        with:
          ref: main

      - name: Login to container registry provider
        uses: docker/login-action@v2
        with:
          registry: ${{ env.CONTAINER_REGISTRY }}
          # GitLab
          # username: ${{ secrets.GITLAB_USERNAME }}
          # password: ${{ secrets.GITLAB_TOKEN }}
          # Quay.io
          # username: ${{ secrets.QUAY_USERNAME }}
          # password: ${{ secrets.QUAY_TOKEN }}
          # GitHub
          username: ${{ github.actor }}
          password: ${{ secrets.PUBLISH_TOKEN }}

      - name: Install Cosign GH action
        uses: sigstore/cosign-installer@main
        with:
          cosign-release: 'v1.13.1'

      - name: "Fetch branch name of latest release versions Other software"
        run: |
          curl -sL https://api.github.com/repos/anchore/syft/releases | \
            jq -r ".[].tag_name" | grep -v rc | sort -r -V | head -n 1 | sed -E 's|^v||' > /tmp/syft-latest-branch-name
          echo "SYFT_VERSION=$(cat /tmp/syft-latest-branch-name)" >> $GITHUB_ENV
          curl -sL https://api.github.com/repos/anchore/grype/releases | \
            jq -r ".[].tag_name" | grep -v rc | sort -r -V | head -n 1 | sed -E 's|^v||' > /tmp/grype-latest-branch-name
          echo "GRYPE_VERSION=$(cat /tmp/grype-latest-branch-name)" >> $GITHUB_ENV

      - name: Install Syft
        run: |
          cd /tmp
          curl -L -O -v https://github.com/anchore/syft/releases/download/v${{env.SYFT_VERSION}}/syft_${{env.SYFT_VERSION}}_linux_amd64.deb
          curl -L -O -v https://github.com/anchore/syft/releases/download/v${{env.SYFT_VERSION}}/syft_${{env.SYFT_VERSION}}_checksums.txt
          sha256sum -c syft_${{env.SYFT_VERSION}}_checksums.txt --ignore-missing
          sudo dpkg -i syft_${{env.SYFT_VERSION}}_linux_amd64.deb

      - name: Set the SBOM env variable for use by later steps
        run: |
          echo "ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT=container-sbom.json" >> $GITHUB_ENV

      - name: Use Syft to generate the SBOM files
        # ## syft -v "registry.gitlab.com/patrickdung/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}" -o json > ./${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
        run: |
          syft -v ${{matrix.platform_image_uri}} -o json > ./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}

      - name: Upload SBOM files to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.PUBLISH_TOKEN }}
          file: ./*${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
          file_glob: true
          tag: ${{ env.REMOTE_BRANCH_NAME }}
          overwrite: true

      # Cannot download if it's not generated as artifact
      # - name: Download the container SBOM artifact
      #  uses: actions/download-artifact@v2
      #  with:
      #    #name: container-sbom.spdx.json
      #    name: ${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}

      - name: Create SBOM attestation
        run: |
          # Create SBOM attestation and push it to the container registry
          echo -n "${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }}" | \
            cosign attest --predicate "${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}" \
            --key <(echo -n "${{ secrets.COSIGN_PRIVATE_KEY }}") \
            "${{ matrix.platform_image_uri }}"

      - name: Install Grype
        run: |
          cd /tmp
          curl -L -O -v https://github.com/anchore/grype/releases/download/v${{env.GRYPE_VERSION}}/grype_${{env.GRYPE_VERSION}}_linux_amd64.deb
          curl -L -O -v https://github.com/anchore/grype/releases/download/v${{env.GRYPE_VERSION}}/grype_${{env.GRYPE_VERSION}}_checksums.txt
          sha256sum -c grype_${{env.GRYPE_VERSION}}_checksums.txt --ignore-missing
          sudo dpkg -i grype_${{env.GRYPE_VERSION}}_linux_amd64.deb

      - name: Scan container by Grype
        run: |
          # May set a severity threshold for failing the build
          grype sbom:./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}} -o json > ./${{matrix.arch}}-container-vulnerabilities-report-grype.json
          grype sbom:./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}} -o table > ./${{matrix.arch}}-container-vulnerabilities-report-grype-table.txt

      - name: Upload Grype reports to artifacts
        uses: actions/upload-artifact@v2
        with:
          path: ./*vulnerabilities-report-grype*
          name: "Vulnerabilities reports by Grype"

      - name: Upload Grype reports to release
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.PUBLISH_TOKEN }}
          file: ./*vulnerabilities-report-grype*
          file_glob: true
          tag: ${{ env.REMOTE_BRANCH_NAME }}
          overwrite: true

      - name: Scan container with Trivy
        uses: aquasecurity/trivy-action@master
        id: scan-by-trivy
        with:
          image-ref: '${{matrix.platform_image_uri}}'
          format: 'template'
          template: '@/contrib/sarif.tpl'
          output: '${{matrix.arch}}-container-trivy-results.sarif'
          severity: 'CRITICAL,HIGH'

      - name: Upload Trivy SARIF report to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: '${{matrix.arch}}-container-trivy-results.sarif'
          category: trivy-${{matrix.arch}}

      - name: Scan container by Anchore
        uses: anchore/scan-action@v3
        id: scan-by-anchore
        with:
          fail-build: false
          image: "${{matrix.platform_image_uri}}"
          acs-report-enable: true

      - name: Rename results.sarif with architecture name
        run: |
          mv results.sarif ${{matrix.arch}}-container-anchore-results.sarif

      - name: Upload Anchore SARIF report to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          # the steps outputs would contain './'
          # sarif_file: anchore-${{matrix.arch}}-${{ steps.scan-by-anchore.outputs.sarif }}
          sarif_file: ./${{matrix.arch}}-container-anchore-results.sarif
          category: anchore-${{matrix.arch}}

      - name: Scan container by Snyk
        continue-on-error: true
        uses: snyk/actions/docker@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_API_TOKEN }}
        with:
          image: ${{matrix.platform_image_uri}}
          args: --file=Dockerfile

      - name: Rename results.sarif with architecture name
        run: |
          mv snyk.sarif ${{matrix.arch}}-container-snyk-results.sarif

      - name: Upload result to GitHub Code Scanning security tab
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: ${{matrix.arch}}-container-snyk-results.sarif

#      - name: Sysdig Secure Inline Scan
#        id: scan
#        uses: sysdiglabs/scan-action@v3
#        with:
#          # Tag of the image to analyse
#          image-tag: "${{matrix.platform_image_uri}}"
#          sysdig-secure-token: ${{ secrets.SYSDIG_SECURE_TOKEN}}
#          # Sysdig secure endpoint. Please read: https://docs.sysdig.com/en/docs/administration/saas-regions-and-ip-ranges/
#          sysdig-secure-url: https://app.au1.sysdig.com
#          dockerfile-path: ./Dockerfile
#          input-type: docker-daemon
#          ignore-failed-scan: true
#          # Sysdig inline scanner requires privileged rights
#          run-as-user: root
#
#      - name: Rename Sysdig sarif with architecture name
#        run: |
#          mv ${{ steps.scan.outputs.sarifReport }} ${{matrix.arch}}-container-sysdig-results.sarif
#
#      - name: Upload Sysdig SARIF report to GitHub Security tab
#        uses: github/codeql-action/upload-sarif@v2
#        if: always()
#        with:
#          sarif_file: '${{matrix.arch}}-container-sysdig-results.sarif'
#          category: sysdig-${{matrix.arch}}

      # - name: Inspect SARIF report(s)
      #  run: |
      #    echo ${{matrix.arch}}
      #    cat ${{matrix.arch}}-container-trivy-results.sarif
      #    cat ${{matrix.arch}}-container-anchore-results.sarif

      - name: Upload SARIF reports to artifacts
        uses: actions/upload-artifact@v2
        with:
            # #${{matrix.arch}}-trivy-results.sarif
            # #${{ steps.scan-by-anchore.outputs.sarif }}
          name: "SARIF reports when containers are built"
          path: |
            ./*.sarif