Skip to content

New release version v1.4.0-rc.2 #149

New release version v1.4.0-rc.2

New release version v1.4.0-rc.2 #149

# SPDX-License-Identifier: Apache-2.0
#
# Copyright (c) 2021 Patrick Dung
name: Pre-release using cross build
on:
push:
branches:
- dev
# paths:
# - 'release-versions/*'
env:
CARGO_TERM_COLOR: always
# CONTAINER_REGISTRY: quay.io
# CONTAINER_REPOSITORY: patrickdung/docker-images-meilisearch
CONTAINER_REGISTRY: ghcr.io
# ## GH Repository must be lower case, when using in GH action
CONTAINER_REPOSITORY: ${{ github.repository }}
# ## CONTAINER_REPOSITORY: patrickdung/meilisearch-crossbuild
ORIGINAL_SOURCECODE_URL: https://github.com/meilisearch/MeiliSearch
jobs:
init-env:
name: Set env outputs
runs-on: ubuntu-latest
continue-on-error: true
permissions:
actions: none
checks: none
contents: none
deployments: none
issues: none
packages: none
pull-requests: none
repository-projects: none
security-events: none
statuses: none
# id-token: write # needed for signing the images with GitHub OIDC **not production ready**
outputs:
container_registry_base_uri: ${{ steps.set-env.outputs.container_registry_base_uri }}
steps:
- name: Set env for later jobs
id: set-env
run: |
echo "container_registry_base_uri=$(echo $CONTAINER_REGISTRY/$(echo $CONTAINER_REPOSITORY|tr 'A-Z' 'a-z'))" >> $GITHUB_OUTPUT
publish-with-crossbuild:
name: Publish to Github
needs: [init-env]
runs-on: ${{ matrix.os }}
continue-on-error: true
strategy:
fail-fast: false
matrix:
include:
- build: aarch64
## ubuntu-18.04/20.04 failed to cross build glibc with meili v0.30 and ld gold
## os: ubuntu-18.04
os: ubuntu-20.04
target: aarch64-unknown-linux-gnu
linker: gcc-aarch64-linux-gnu
use-cross: true
asset_name: meilisearch-linux-aarch64
- build: linux
os: ubuntu-20.04
target: x86_64-unknown-linux-gnu
use-cross: false
asset_name: meilisearch-linux-x86_64
- build: aarch64
os: ubuntu-20.04
target: aarch64-unknown-linux-musl
linker: gcc-aarch64-linux-gnu
use-cross: true
asset_name: meilisearch-linux-aarch64-musl
- build: linux
os: ubuntu-20.04
target: x86_64-unknown-linux-musl
use-cross: true
asset_name: meilisearch-linux-x86_64-musl
outputs:
remote_branch_name: ${{ steps.get-remote-branch-name.outputs.remote_branch_name }}
steps:
- name: Get branch name of latest pre-release version of Meili
id: get-remote-branch-name
run: |
curl -sL https://api.github.com/repos/meilisearch/MeiliSearch/releases | \
jq -r "map(select(.prerelease)) | first | .tag_name" > /tmp/meilisearch-latest-branch-name
echo "REMOTE_BRANCH_NAME=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_ENV
echo "remote_branch_name=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_OUTPUT
- name: Checkout repository of offical repo for compiling
uses: actions/checkout@v2
with:
repository: meilisearch/MeiliSearch
ref: ${{ env.REMOTE_BRANCH_NAME }}
- name: Installing Rust toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
profile: minimal
target: ${{ matrix.target }}
override: true
- name: APT update
run: |
sudo apt update
- name: Install target specific tools
if: matrix.use-cross
run: |
sudo apt-get install -y ${{ matrix.linker }}
- name: Configure target aarch64 GNU
if: matrix.target == 'aarch64-unknown-linux-gnu'
## Environment variable is not passed using env:
## LD gold won't work with MUSL
# env:
# JEMALLOC_SYS_WITH_LG_PAGE: 16
# RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
run: |
echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
# echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
- name: Configure target aarch64 MUSL
if: matrix.target == 'aarch64-unknown-linux-musl'
# env:
# JEMALLOC_SYS_WITH_LG_PAGE: 16
run: |
sudo apt-get install -y musl-tools
echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
- name: Configure target x86_64 MUSL
if: matrix.target == 'x86_64-unknown-linux-musl'
run: |
sudo apt-get install -y musl-tools
- name: Cargo build
uses: actions-rs/cargo@v1
with:
command: build
use-cross: ${{ matrix.use-cross }}
args: --release --target ${{ matrix.target }}
# Strip debuginfo for target aarch64 GNU
# MUSL binaries are static linked
- name: Strip debuginfo for target aarch64 GNU
if: matrix.target == 'aarch64-unknown-linux-gnu'
run: |
/usr/bin/aarch64-linux-gnu-strip --strip-debug --target=elf64-littleaarch64 target/${{ matrix.target }}/release/meilisearch -o target/${{ matrix.target }}/release/meilisearch-stripped
- name: Strip debuginfo for target x86_64 GNU
if: matrix.target == 'x86_64-unknown-linux-gnu'
run: |
strip --strip-debug target/${{ matrix.target }}/release/meilisearch -o target/${{ matrix.target }}/release/meilisearch-stripped
- name: Create checksum file for the binaries
run: |
cd target/${{ matrix.target }}/release
sha256sum meilisearch | awk '{print $1, "${{matrix.asset_name}}"}' > ${{matrix.asset_name}}.sha256sum
if [ -e meilisearch-stripped ]; then
sha256sum meilisearch-stripped | awk '{print $1, "${{matrix.asset_name}}-stripped"}' > ${{matrix.asset_name}}-stripped.sha256sum
fi
- name: List target output files
run: ls -lR ./target
- name: Upload the binary to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
# repo_token: ${{ secrets.GITHUB_TOKEN }}
file: target/${{ matrix.target }}/release/meilisearch
asset_name: ${{ matrix.asset_name }}
tag: ${{ env.REMOTE_BRANCH_NAME }}
prerelease: true
overwrite: true
- name: Upload stripped binary to release (aarch64/x86_64 GNU only)
if: matrix.target == 'aarch64-unknown-linux-gnu' || matrix.target == 'x86_64-unknown-linux-gnu'
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
# repo_token: ${{ secrets.GITHUB_TOKEN }}
file: target/${{ matrix.target }}/release/meilisearch-stripped
asset_name: ${{ matrix.asset_name }}-stripped
tag: ${{ env.REMOTE_BRANCH_NAME }}
prerelease: true
overwrite: true
- name: Upload checksum files to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
file: target/${{ matrix.target }}/release/*.sha256sum
file_glob: true
tag: ${{ env.REMOTE_BRANCH_NAME }}
prerelease: true
overwrite: true
build-docker-image:
# needs: publish-with-crossbuild
needs: [init-env]
name: Build Docker Images
runs-on: ubuntu-latest
continue-on-error: true
permissions:
actions: none
checks: none
contents: read
deployments: none
issues: none
packages: write
pull-requests: none
repository-projects: none
# GH action/scanners for sarif reports
security-events: write
statuses: none
# id-token: write # needed for signing the images with GitHub OIDC **not production ready**
outputs:
container_digest_amd64: ${{ steps.get-container-digest-amd64.outputs.container_digest }}
container_digest_arm64: ${{ steps.get-container-digest-arm64.outputs.container_digest }}
services:
registry:
image: registry:2
ports:
- 5000:5000
steps:
- name: "Fetch branch name of latest version of Meili"
run: |
curl -sL https://api.github.com/repos/meilisearch/MeiliSearch/releases | \
jq -r "map(select(.prerelease)) | first | .tag_name" > /tmp/meilisearch-latest-branch-name
echo "REMOTE_BRANCH_NAME=$(cat /tmp/meilisearch-latest-branch-name)" >> $GITHUB_ENV
- name: Checkout repository
uses: actions/checkout@v2
with:
repository: patrickdung/MeiliSearch-crossbuild
ref: dev
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
with:
driver-opts: network=host
- name: Login to Registry provider
uses: docker/login-action@v1
with:
registry: ${{ env.CONTAINER_REGISTRY }}
# Quay.io
# username: ${{ secrets.QUAY_USERNAME }}
# password: ${{ secrets.QUAY_TOKEN }}
# GitHub
username: ${{ github.actor }}
password: ${{ secrets.PUBLISH_TOKEN }}
- name: Install Cosign GH action
uses: sigstore/cosign-installer@main
with:
cosign-release: 'v1.13.1'
# Distroless don't have package managers
# - name: Verify Distroless image
# run: |
# cosign verify --key cosign.pub gcr.io/distroless/base-debian11:latest-amd64
# cosign verify --key cosign.pub gcr.io/distroless/base-debian11:latest-arm64
- name: Build and push
uses: docker/build-push-action@v2
env:
DOCKER_CONTENT_TRUST: 1
with:
context: .
build-args: |
MEILISEARCH_VERSION=${{ env.REMOTE_BRANCH_NAME }}
SOURCE_BINARY_BASEURL=https://github.com/patrickdung/MeiliSearch-crossbuild/releases/download
LABEL_IMAGE_URL=${{ env.ORIGINAL_SOURCECODE_URL }}
LABEL_IMAGE_SOURCE=https://github.com/${{ github.repository }}
platforms: linux/amd64,linux/arm64
push: true
# registry.gitlab.com/patrickdung/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}
tags: |
${{ needs.init-env.outputs.container_registry_base_uri }}:${{ env.REMOTE_BRANCH_NAME }}
localhost:5000/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}
- name: "Get the digest of container (amd64)"
id: get-container-digest-amd64
run: |
skopeo inspect --raw docker://${{needs.init-env.outputs.container_registry_base_uri}}:${{env.REMOTE_BRANCH_NAME}} | \
jq -r '.manifests[] | select(.platform .architecture=="amd64" and .platform .os=="linux") | .digest' > /tmp/container-digest-amd64
echo "container_digest=$(cat /tmp/container-digest-amd64)" >> $GITHUB_OUTPUT
- name: "Get the digest of container (arm64)"
id: get-container-digest-arm64
run: |
skopeo inspect --raw docker://${{needs.init-env.outputs.container_registry_base_uri}}:${{env.REMOTE_BRANCH_NAME}} | \
jq -r '.manifests[] | select(.platform .architecture=="arm64" and .platform .os=="linux") | .digest' > /tmp/container-digest-arm64
echo "container_digest=$(cat /tmp/container-digest-arm64)" >> $GITHUB_OUTPUT
- name: Use Cosign to sign the image recursively
run: |
# Sign the multiarch images, the signature is also pushed to the registry
echo -n "${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }}" | \
cosign sign --recursive --key <(echo -n "${{ secrets.COSIGN_PRIVATE_KEY }}") \
"${{ needs.init-env.outputs.container_registry_base_uri }}:${{ env.REMOTE_BRANCH_NAME }}"
build-sbom-and-scanning:
needs: [init-env, publish-with-crossbuild, build-docker-image]
name: Build SBOM and image scanning
runs-on: ubuntu-latest
continue-on-error: true
permissions:
actions: none
checks: none
contents: read
deployments: none
issues: none
packages: write
pull-requests: none
repository-projects: none
# GH action/scanners for sarif reports
security-events: write
statuses: none
# id-token: write # needed for signing the images with GitHub OIDC **not production ready**
strategy:
max-parallel: 1
fail-fast: false
matrix:
include:
- arch: amd64
platform_image_uri: "${{needs.init-env.outputs.container_registry_base_uri}}@${{ needs.build-docker-image.outputs.container_digest_amd64 }}"
- arch: arm64
platform_image_uri: "${{needs.init-env.outputs.container_registry_base_uri}}@${{ needs.build-docker-image.outputs.container_digest_arm64 }}"
steps:
- name: Set env REMOTE_BRANCH_NAME
run: |
echo "REMOTE_BRANCH_NAME=${{ needs.publish-with-crossbuild.outputs.remote_branch_name }}" >> $GITHUB_ENV
- name: Checkout this repository
# some vuln scanner want to have the Dockerfile
uses: actions/checkout@v2
with:
ref: dev
- name: Login to Registry provider
uses: docker/login-action@v1
with:
registry: ${{ env.CONTAINER_REGISTRY }}
# username: ${{ secrets.QUAY_USERNAME }}
# password: ${{ secrets.QUAY_TOKEN }}
username: ${{ github.actor }}
password: ${{ secrets.PUBLISH_TOKEN }}
- name: Install Cosign GH action
uses: sigstore/cosign-installer@main
# with:
# cosign-release: 'v1.4.1'
- name: "Fetch branch name of latest release versions Other software"
run: |
curl -sL https://api.github.com/repos/anchore/syft/releases | \
jq -r ".[].tag_name" | grep -v rc | sort -r -V | head -n 1 | sed -E 's|^v||' > /tmp/syft-latest-branch-name
echo "SYFT_VERSION=$(cat /tmp/syft-latest-branch-name)" >> $GITHUB_ENV
curl -sL https://api.github.com/repos/anchore/grype/releases | \
jq -r ".[].tag_name" | grep -v rc | sort -r -V | head -n 1 | sed -E 's|^v||' > /tmp/grype-latest-branch-name
echo "GRYPE_VERSION=$(cat /tmp/grype-latest-branch-name)" >> $GITHUB_ENV
- name: skopeo inspect
run: |
skopeo inspect --raw docker://${{matrix.platform_image_uri}}
# Grype can't read it (sbom action uses syft v0.26 and default is spdx-json)
# - name: Use SBOM action to generate the SBOM
# uses: anchore/sbom-action@v0
# with:
# image: registry.gitlab.com/patrickdung/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}
# artifact-name: container-sbom.spdx.json
- name: Install Syft
run: |
cd /tmp
curl -L -O -v https://github.com/anchore/syft/releases/download/v${{env.SYFT_VERSION}}/syft_${{env.SYFT_VERSION}}_linux_amd64.deb
curl -L -O -v https://github.com/anchore/syft/releases/download/v${{env.SYFT_VERSION}}/syft_${{env.SYFT_VERSION}}_checksums.txt
sha256sum -c syft_${{env.SYFT_VERSION}}_checksums.txt --ignore-missing
sudo dpkg -i syft_${{env.SYFT_VERSION}}_linux_amd64.deb
- name: Set the SBOM env variable for use by later steps
run: |
echo "ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT=container-sbom.json" >> $GITHUB_ENV
- name: Use Syft to generate the SBOM files
run: |
##syft -v "registry.gitlab.com/patrickdung/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}" -o json > ./${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
syft -v ${{matrix.platform_image_uri}} -o json > ./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
# Not working and artifact is not published
# - name: Publish SBOM to release package
# uses: anchore/sbom-action/publish-sbom@v0
# with:
# ##sbom-artifact-match: .*\\.spdx\\.json$
# ## ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT: container-sbom.spdx.json
# sbom-artifact-match: ${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
# github-token: ${{secrets.PUBLISH_TOKEN}}
- name: Upload SBOM files to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
file: ./*${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
file_glob: true
tag: ${{ env.REMOTE_BRANCH_NAME }}
prerelease: true
overwrite: true
# Cannot download if it's not generated as artifact
# - name: Download the container SBOM artifact
# uses: actions/download-artifact@v2
# with:
# #name: container-sbom.spdx.json
# name: ${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
- name: Create SBOM attestation
# env:
# SBOM_FILE: ./container-sbom.spdx.json
# SBOM_FILE: ${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}
run: |
# CONTAINER_IMAGE=${{matrix.platform_image_uri}}
# Create SBOM attestation and push it to the container registry
# cosign attest --predicate "$SBOM_FILE" --key ./cosign.key "$CONTAINER_IMAGE"
echo -n "${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }}" | \
cosign attest --predicate "${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}}" \
--key <(echo -n "${{ secrets.COSIGN_PRIVATE_KEY }}") \
"${{matrix.platform_image_uri}}"
- name: Install Grype
run: |
cd /tmp
curl -L -O -v https://github.com/anchore/grype/releases/download/v${{env.GRYPE_VERSION}}/grype_${{env.GRYPE_VERSION}}_linux_amd64.deb
curl -L -O -v https://github.com/anchore/grype/releases/download/v${{env.GRYPE_VERSION}}/grype_${{env.GRYPE_VERSION}}_checksums.txt
sha256sum -c grype_${{env.GRYPE_VERSION}}_checksums.txt --ignore-missing
sudo dpkg -i grype_${{env.GRYPE_VERSION}}_linux_amd64.deb
- name: Scan container by Grype
run: |
# May set a severity threshold for failing the build
grype sbom:./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}} -o json > ./${{matrix.arch}}-container-vulnerabilities-report-grype.json
grype sbom:./${{matrix.arch}}-${{env.ANCHORE_SBOM_ACTION_PRIOR_ARTIFACT}} -o table > ./${{matrix.arch}}-container-vulnerabilities-report-grype-table.txt
- name: Upload Grype reports to artifacts
uses: actions/upload-artifact@v2
with:
path: ./*vulnerabilities-report-grype*
name: "Vulnerabilities reports by Grype"
- name: Upload Grype reports to release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.PUBLISH_TOKEN }}
file: ./*vulnerabilities-report-grype*
file_glob: true
tag: ${{ env.REMOTE_BRANCH_NAME }}
prerelease: true
overwrite: true
- name: Scan container with Trivy
uses: aquasecurity/trivy-action@master
id: scan-by-trivy
with:
# ##image-ref: 'registry.gitlab.com/patrickdung/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}'
image-ref: '${{matrix.platform_image_uri}}'
format: 'template'
template: '@/contrib/sarif.tpl'
output: '${{matrix.arch}}-container-trivy-results.sarif'
severity: 'CRITICAL,HIGH'
- name: Upload Trivy SARIF report to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: '${{matrix.arch}}-container-trivy-results.sarif'
category: trivy-${{matrix.arch}}
- name: Scan container by Anchore
uses: anchore/scan-action@v3
id: scan-by-anchore
with:
fail-build: false
# image: "localhost:5000/docker-images/meilisearch:${{ env.REMOTE_BRANCH_NAME }}"
image: "${{matrix.platform_image_uri}}"
acs-report-enable: true
- name: Rename results.sarif with architecture name
run: |
mv results.sarif ${{matrix.arch}}-container-anchore-results.sarif
- name: Upload Anchore SARIF report to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
# the steps outputs would contain './'
# ##sarif_file: ${{matrix.arch}}-${{ steps.scan-by-anchore-${{matrix.arch}}.outputs.sarif }}
# sarif_file: anchore-${{matrix.arch}}-${{ steps.scan-by-anchore.outputs.sarif }}
sarif_file: ./${{matrix.arch}}-container-anchore-results.sarif
category: anchore-${{matrix.arch}}
- name: Scan container by Snyk
continue-on-error: true
uses: snyk/actions/docker@master
env:
SNYK_TOKEN: ${{ secrets.SNYK_API_TOKEN }}
with:
image: ${{matrix.platform_image_uri}}
args: --file=Dockerfile
- name: Rename results.sarif with architecture name
run: |
mv snyk.sarif ${{matrix.arch}}-container-snyk-results.sarif
- name: Upload result to GitHub Code Scanning security tab
uses: github/codeql-action/upload-sarif@v2
if: always()
with:
sarif_file: ${{matrix.arch}}-container-snyk-results.sarif
# - name: Sysdig Secure Inline Scan
# id: scan
# uses: sysdiglabs/scan-action@v3
# with:
# # Tag of the image to analyse
# image-tag: "${{matrix.platform_image_uri}}"
# sysdig-secure-token: ${{ secrets.SYSDIG_SECURE_TOKEN}}
# # Sysdig secure endpoint. Please read: https://docs.sysdig.com/en/docs/administration/saas-regions-and-ip-ranges/
# sysdig-secure-url: https://app.au1.sysdig.com
# dockerfile-path: ./Dockerfile
# input-type: docker-daemon
# ignore-failed-scan: true
# # Sysdig inline scanner requires privileged rights
# run-as-user: root
#
# - name: Rename Sysdig sarif with architecture name
# run: |
# mv ${{ steps.scan.outputs.sarifReport }} ${{matrix.arch}}-container-sysdig-results.sarif
#
# - name: Upload Sysdig SARIF report to GitHub Security tab
# uses: github/codeql-action/upload-sarif@v2
# if: always()
# with:
# sarif_file: '${{matrix.arch}}-container-sysdig-results.sarif'
# category: sysdig-${{matrix.arch}}
# - name: Inspect SARIF report(s)
# run: |
# echo ${{matrix.arch}}
# cat ${{matrix.arch}}-container-trivy-results.sarif
# cat ${{matrix.arch}}-container-anchore-results.sarif
- name: Upload SARIF reports to artifacts
uses: actions/upload-artifact@v2
with:
name: "SARIF reports when containers are built"
# ${{matrix.arch}}-container-trivy-results.sarif
# ${{ steps.scan-by-anchore.outputs.sarif }}
path: |
./*.sarif