Cache Sources and build Artifact Libraries per OS

Multiplatform build and upload #413

Workflow file for this run

.github/workflows/build-targets.yml at 5d8a80e

	# This is a workflow to cache the sources and build all the artifacts for target OSs

	name: Cache Sources and build Artifact Libraries per OS

	on:
	workflow_dispatch:

	jobs:
	setup-src-cache:
	runs-on: ubuntu-latest
	outputs:
	THIRD_PARTY_CACHE_NAME_KEY: ${{ steps.tp-check.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	FOUND_THIRD_PARTY_CURRENT: ${{ steps.tp-check.outputs.FOUND_THIRD_PARTY_CURRENT }}
	CACHE_MATRIX: ${{ steps.set-cache-matrix.outputs.CACHE_MATRIX }}
	NEEDS_CACHE: ${{ steps.set-cache-matrix.outputs.NEEDS_CACHE }}
	CACHE_MULTI_PARAMETER_NAME_KEYS: ${{ steps.set-cache-matrix.outputs.CACHE_MULTI_PARAMETER_NAME_KEYS }}
	CACHE_MULTI_PARAMETER_PATHS: ${{ steps.set-cache-matrix.outputs.CACHE_MULTI_PARAMETER_PATHS }}
	steps:
	- name: Check out main third party without submodules
	uses: actions/checkout@v4
	with:
	submodules: false

	- name: Authenticate GitHub CLI
	run: \|
	echo $GH_TOKEN \| gh auth login --with-token
	gh auth status
	env:
	GH_TOKEN: ${{ secrets.GH_TOKEN }}

	- name: Install GH cli extension actions-cache
	run: \|
	gh extension install actions/gh-actions-cache

	# get currently authenticated user rate limit info
	rateRemaining=`gh api rate_limit --jq ".rate \| .remaining"`
	echo "Rate limit remaining is $rateRemaining"

	echo "REPO=${{ github.repository }}" >> $GITHUB_ENV
	echo "BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
	env:
	GH_TOKEN: ${{ secrets.GH_TOKEN }}

	- name: Find cache and clean out old caches of `thirdparty`
	id: tp-check
	run: \|
	echo "Fetching list of thirdparty cache keys to check if any old ones need deleting"
	cacheList=$(gh cache list -k ${BRANCH}/thirdparty --json key --jq ".[].key")

	longSHA="${{ github.sha }}"
	shortSHA="${longSHA:0:7}"
	tpCacheNameSHA="$BRANCH/thirdparty-$shortSHA"

	notMatched="$(comm -3 --output-delimiter="" <(echo -en "$tpCacheNameSHA") <(echo -en "$cachesList"))"
	oldCaches="$(comm -13 --output-delimiter="" <(echo -en "$tpCacheNameSHA") <(echo -en "$notMatched"))"
	notCached="$(comm -12 --output-delimiter="" <(echo -en "$tpCacheNameSHA") <(echo -en "$notMatched"))"

	echo -e "notMatched: $notMatched"
	echo -e "oldCaches: $oldCaches"
	echo -e "notCached: $notCached"

	foundThirdPartyCurrent="true"
	if [[ $notCached == "$tpCacheNameSHA" ]]; then
	foundThirdPartyCurrent="false"
	fi

	# Set the field separator to new line
	IFS=$'\n'
	for oldCache in $oldCaches; do
	echo -e "Deleting old cache $oldCache"
	gh actions-cache delete $oldCache -R $REPO -B $BRANCH --confirm
	done

	echo "FOUND_THIRD_PARTY_CURRENT=$foundThirdPartyCurrent" >> $GITHUB_OUTPUT
	echo "THIRD_PARTY_CACHE_NAME_KEY=$tpCacheNameSHA" >> $GITHUB_OUTPUT
	shell: bash
	working-directory: ${{github.workspace}}

	- name: Cache source files
	if: ( steps.tp-check.outputs.FOUND_THIRD_PARTY_CURRENT == 'false' )
	uses: GeniusVentures/[email protected]
	id: cache-source-directory
	with:
	key: ${{ steps.tp-check.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	enableCrossOsArchive: true
	path: \|
	!.git/**
	./**

	- name: Set matrix of src submodules that need updating
	id: set-cache-matrix
	run: \|
	echo "Fetching list of cache keys to check if any old ones need deleting"
	cacheNames=""
	while read -r mode stage longsha location; do
	shortSHA="${longsha:0:7}"
	cacheNames+="s/$BRANCH/$location-$shortSHA\n"
	done <<< $(git ls-tree -r HEAD \| grep "160000 commit")

	cacheNames="$(echo -en "$cacheNames" \| sort)"
	cachesList=$(gh api -H "Accept: application/vnd.github+json" --paginate repos/GeniusVentures/thirdparty/actions/caches?key="s/${BRANCH}" \| jq -r ".actions_caches[].key" \| sort)

	notMatched="$(comm -3 --output-delimiter="" <(echo -en "$cacheNames") <(echo -en "$cachesList"))"
	oldCaches="$(comm -13 --output-delimiter="" <(echo -en "$cacheNames") <(echo -en "$notMatched"))"
	notCached="$(comm -12 --output-delimiter="" <(echo -en "$cacheNames") <(echo -en "$notMatched"))"

	# Matrix of source code to cache
	cacheMatrix="[]"

	# Set the field separator to new line
	IFS=$'\n'
	for toCache in $notCached; do
	len=${#toCache}
	startPos=$((len - 7))
	lenBranch=${#BRANCH}
	matrixItem=$(jq -n \
	--arg cacheName "${toCache}" \
	--arg name "${toCache:$((2 + lenBranch + 1)):$((len - 10 - lenBranch - 1))}" \
	'{name: $name, cacheName: $cacheName}'
	)

	cacheMatrix=$(jq -c --argjson obj "$matrixItem" '. + [$obj]' <<<"$cacheMatrix")
	done

	echo "Cache matrix:"
	jq <<<"$cacheMatrix"

	# Delete old caches
	for oldCache in $oldCaches; do
	echo "Deleting old cache $oldCache"
	gh actions-cache delete $oldCache -R $REPO -B $BRANCH --confirm
	done

	echo "CACHE_MATRIX=$cacheMatrix" >> $GITHUB_OUTPUT
	if [[ "$notCached" != "" ]]; then
	echo "NEEDS_CACHE=true" >> $GITHUB_OUTPUT
	else
	echo "NEEDS_CACHE=false" >> $GITHUB_OUTPUT
	fi

	cacheNameKeysContent="[ "
	cachePathsContent="[ "
	first='true'
	for cacheName in $cacheNames; do
	len=${#cacheName}
	startPos=$((len - 7))
	shortSHA="${cacheName:$startPos:7}"
	lenBranch=${#BRANCH}
	# - 10 - lenBranch - 1 for s/$BRANCH/ + '-' + shortSHA
	name="${cacheName:$((2 + lenBranch + 1)):$((len - 10 - lenBranch - 1))}"
	if [[ "$first" != "true" ]]; then
	cacheNameKeysContent+=", "
	cachePathsContent+=", "
	fi
	first='false'
	cacheNameKeysContent+="\"$cacheName\""
	cachePathsContent+="[\"!./$name/.git/\", \"./$name/\"]"
	done

	cacheNameKeysContent+=" ]"
	echo "CACHE_MULTI_PARAMETER_NAME_KEYS=$cacheNameKeysContent" >> $GITHUB_OUTPUT

	cachePathsContent+=" ]"
	echo "CACHE_MULTI_PARAMETER_PATHS=$cachePathsContent" >> $GITHUB_OUTPUT

	shell: bash
	working-directory: ${{github.workspace}}

	cache-src:
	needs: setup-src-cache
	if: (needs.setup-src-cache.outputs.NEEDS_CACHE == 'true')
	runs-on: ubuntu-latest
	strategy:
	fail-fast: true
	matrix:
	include: ${{ fromJson(needs.setup-src-cache.outputs.CACHE_MATRIX) }}
	steps:
	- name: Setup
	run: \|
	sudo apt install zstd -y

	- name: Cache third party directory
	uses: GeniusVentures/cache-multi/[email protected]
	id: cache-tp-source-directory
	with:
	key: ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	enableCrossOsArchive: true
	path: \|
	!.git/**
	./**

	- if: (steps.cache-tp-source-directory.outputs.cache-hit != 'true')
	run: \|
	echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
	exit 1

	- name: Git clone submodule source code
	run: \|
	echo "Cloning source for ${{ matrix.name }}"
	git submodule update --init --recommend-shallow --recursive -- ${{ matrix.name }}
	shell: bash
	working-directory: ${{github.workspace}}

	- name: Cache submodule source files
	id: cache-source-directory
	uses: GeniusVentures/[email protected]
	with:
	key: ${{ matrix.cacheName }}
	enableCrossOsArchive: true
	path: \|
	!./${{ matrix.name }}/.git/**
	./${{ matrix.name }}/**

	create-targets-matrix:
	needs: [setup-src-cache, cache-src]
	if: \|
	always() &&
	needs.setup-src-cache.result == 'success' &&
	(needs.cache-src.result == 'success' \|\| needs.cache-src.result == 'skipped')
	runs-on: ubuntu-latest
	outputs:
	TARGET_NAMES: ${{ steps.create-target-info.outputs.TARGET_NAMES }}
	TARGET_SHAS: ${{ steps.create-target-info.outputs.TARGET_SHAS }}
	TARGET_DIRECTORIES: ${{ steps.create-target-info.outputs.TARGET_DIRECTORIES }}
	TARGET_REPO_NAMES: ${{ steps.create-target-info.outputs.TARGET_REPO_NAMES }}
	steps:
	- name: Setup
	run: sudo apt install zstd -y

	- name: Restore cache third party directory
	uses: GeniusVentures/cache-multi/[email protected]
	id: cache-tp-source-directory
	with:
	key: ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	enableCrossOsArchive: true
	path: \|
	!.git/**
	./**

	- if: (steps.cache-tp-source-directory.outputs.cache-hit != 'true')
	run: \|
	echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
	exit 1

	- name: Restore submodule source files
	id: cache-source-directory
	uses: GeniusVentures/cache-multi/[email protected]
	with:
	multi-keys: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_NAME_KEYS }}
	enableCrossOsArchive: true
	paths: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_PATHS }}

	- name: Create base target library names for target cache to create
	id: create-target-info
	run: \|
	cmake -S build/Linux -B build/Linux/Release -DCMAKE_BUILD_TYPE=Release

	cd build/Linux/Release/CMakeFiles
	declare -A cmakeBuildDir
	while IFS=',' read -r cmakeName buildDir; do
	cmakeBuildDir["$cmakeName"]="$buildDir"
	done < <(grep -m1 -R --include "build.make" -e ".-complete:.-build" * \|
	sed -E -n 's/^(.)\.dir.-complete: ([^/]?)\/src./\1,\2/p')

	cd ..
	declare -A cmakeNamesRepos
	for key in "${!cmakeBuildDir[@]}"; do
	sourceDir=$(find . -type f -name "${key}-source_dirinfo.txt" -exec sed -n "s\|^source_dir=$GITHUB_WORKSPACE\/$.*$\|\1\|p" {} \;)
	cmakeNamesRepos["$key"]=$sourceDir
	echo "cmakeNameRepos = ${sourceDir}"
	done

	declare -A gitLibModules
	cd $GITHUB_WORKSPACE
	while IFS=',' read -r longSHA sourceDir; do
	gitLibModules["$sourceDir"]="$longSHA"
	done < <(git ls-tree -r HEAD \| sed -E -n 's/^.*(commit\|tree)\s+([a-f0-9]+)\s+(\S+)/\2,\3/p')

	# Initialize the variables
	cmakeNames=""
	buildDirectoryNames=""
	repoNames=""
	shortSHAs=""

	# Define column widths
	name_width=20
	build_dir_width=20
	source_dir_width=20

	# Print header
	printf "%-${name_width}s \| %-${build_dir_width}s \| %-${source_dir_width}s \| %-${sha_width}s\n" "Name" "Build dir" "Source dir" "SHA"
	for key in "${!cmakeBuildDir[@]}"; do
	buildDir="${cmakeBuildDir[$key]}"
	sourceDir="${cmakeNamesRepos[$key]}"
	shortSHA="${gitLibModules[$sourceDir]:0:7}"
	cmakeNames+="${key}\n"
	buildDirectoryNames+="${buildDir}\n"
	repoNames+="${sourceDir}\n"
	shortSHAs+="${shortSHA}\n"
	printf "%-${name_width}s \| %-${build_dir_width}s \| %-${source_dir_width}s \| %s\n" "${key}" "${buildDir}" "${sourceDir}" "${shortSHA}"
	done

	echo "TARGET_NAMES=$cmakeNames" >> $GITHUB_OUTPUT
	echo "TARGET_SHAS=$shortSHAs" >> $GITHUB_OUTPUT
	echo "TARGET_DIRECTORIES=$buildDirectoryNames" >> $GITHUB_OUTPUT
	echo "TARGET_REPO_NAMES=$repoNames" >> $GITHUB_OUTPUT

	shell: bash
	working-directory: ${{github.workspace}}

	save-target-sources:
	needs: [setup-src-cache, cache-src, create-targets-matrix]
	if: \|
	always() &&
	needs.setup-src-cache.result == 'success' &&
	needs.create-targets-matrix.result == 'success' &&
	(needs.cache-src.result == 'success' \|\| needs.cache-src.result == 'skipped') &&
	needs.create-targets-matrix.result == 'success'
	strategy:
	fail-fast: false
	matrix:
	target-os: [Linux, Windows, OSX, iOS, Android]
	build-type: [Release, Debug]
	runs-on: ubuntu-latest
	steps:
	- name: Setup
	run: \|
	sudo apt install zstd -y

	- name: Create library Keys and Paths to release sources
	id: create-src-cache-keys
	run: \|
	TARGET_OS="${{ matrix.target-os }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	BRANCH=${{ github.ref_name }}
	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	echo "TARGET_OS=${TARGET_OS}" >> $GITHUB_OUTPUT
	echo "BUILD_TYPE=${BUILD_TYPE}" >> $GITHUB_OUTPUT
	echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
	echo "GITHUB_RELEASE_NAME=${GITHUB_RELEASE_NAME}" >> $GITHUB_OUTPUT

	- name: Cache Third Party Directory
	uses: GeniusVentures/cache-multi/[email protected]
	id: cache-tp-source-directory
	with:
	key: ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	enableCrossOsArchive: true
	path: \|
	!.git/**
	./**

	- if: (steps.cache-tp-source-directory.outputs.cache-hit != 'true')
	run: \|
	echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
	exit 1

	- name: Cache Submodule source files without .git
	id: cache-source-directory
	uses: GeniusVentures/cache-multi/[email protected]
	with:
	multi-keys: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_NAME_KEYS }}
	enableCrossOsArchive: true
	paths: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_PATHS }}

	- name: Creating github release tag
	id: create-github-release-tag
	run: \|
	echo "Check if github release tag ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} available."

	set +e
	gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}
	releaseFound=$?
	set -e

	if [[ $releaseFound -ne 0 ]];
	then
	echo "gh release view return value: ${releaseFound}"
	echo "Creating github release with tag: ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}"

	RELEASE_TYPE="--latest"
	if [[ "${{ steps.create-src-cache-keys.outputs.BUILD_TYPE }}" != "Release" ]];
	then
	RELEASE_TYPE="--prerelease"
	fi

	# Create release on github
	gh release create ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} \
	-n "${{ steps.create-src-cache-keys.outputs.BRANCH }} branch" \
	--target "${{ steps.create-src-cache-keys.outputs.BRANCH }}" \
	${RELEASE_TYPE} \
	-t "${{ steps.create-src-cache-keys.outputs.TARGET_OS }} ${{ steps.create-src-cache-keys.outputs.BRANCH }} branch ${{ steps.create-src-cache-keys.outputs.BUILD_TYPE }} build"

	releaseCreated=$?
	if [[ $releaseCreated -ne 0 ]];
	then
	echo "gh release create return value: ${releaseCreated}"
	echo "Failed to create github release with tag: ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}"
	exit 1
	fi
	fi
	env:
	GH_TOKEN: ${{ secrets.GH_TOKEN }}

	- name: Compress and upload targets source folder
	id: compress-targets-source
	working-directory: ${{github.workspace}}
	run: \|
	# build array of library names
	IFS=$'\n' read -d '\|' -ra TARGET_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_NAMES }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_REPO_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_REPO_NAMES }}\|')"

	# Check if release.json file already available
	jsonStr="[]"
	jsonFound=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets --jq "any(.assets[]; .name == \"release.json\")"`
	if [ $? -eq 0 ]
	then
	echo "Downloading release.json file"
	set +e
	gh release download ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --pattern "release.json" --clobber
	if [ $? -eq 0 ]
	then
	echo "Loading release.json file"
	jsonStr=`cat release.json`
	fi
	set -e
	fi

	RELEASE_ASSETS=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets`

	# get currently authenticated user rate limit info
	rateRemaining=`gh api rate_limit --jq ".rate \| .remaining"`
	echo "Rate limit remaining is $rateRemaining"

	# if we don't have at least 50 requests left, wait until reset
	if [[ $rateRemaining -lt 50 ]];
	then
	rateReset=`gh api rate_limit --jq ".rate \| .reset"`
	wait=$((rateReset - `date +%s`));
	echo "Rate limit remaining is $rateRemaining, waiting for $wait seconds to reset"
	while [ $wait -ge 0 ]
	do
	sleep 5
	wait=$((rateReset - `date +%s`));
	echo "Waiting $wait seconds to reset rate limit"
	done
	rateRemaining=`gh api rate_limit --jq ".rate \| .remaining"`
	echo "Rate limit has reset to $rateRemaining requests"
	fi


	for i in "${!TARGET_NAMES[@]}"; do
	TARGET_NAME="${TARGET_NAMES[i]}"
	TARGET_REPO_NAME="${TARGET_REPO_NAMES[i]}"
	TARGET_SHA="${TARGET_SHAS[i]}"
	COMPRESSED_TARGET_SRC="${TARGET_NAME}-src.tar.gz"

	shaFound=$(jq ".[] \| select(.name == \"${TARGET_NAME}\") \| .sha" <<<${jsonStr})
	COMPRESSED_TARGET_SRC_URL=$(jq '.assets[] \| select(.name == "${COMPRESSED_TARGET_SRC}") \| .url' <<<${RELEASE_ASSETS})
	if [ -z "$COMPRESSED_TARGET_SRC_URL" ] \|\| [[ "${shaFound}" != \"${TARGET_SHA}\" ]];
	then
	echo "Compressing ${TARGET_OS} version ${TARGET_SHA} of ${TARGET_REPO_NAME} source folder"
	tar -zcf ${COMPRESSED_TARGET_SRC} ${TARGET_REPO_NAME}
	echo "Uploading target ${COMPRESSED_TARGET_SRC} to github release ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}"
	gh release upload --clobber ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} ${COMPRESSED_TARGET_SRC}
	fi
	done

	build-targets-matrix:
	needs:
	[setup-src-cache, cache-src, create-targets-matrix, save-target-sources]
	if: \|
	always() &&
	needs.setup-src-cache.result == 'success' &&
	needs.create-targets-matrix.result == 'success' &&
	(needs.cache-src.result == 'success' \|\| needs.cache-src.result == 'skipped') &&
	(needs.save-target-sources.result == 'success' \|\| needs.save-target-sources.result == 'skipped')
	strategy:
	fail-fast: false
	matrix:
	build-type: [Release, Debug]
	target-build-data:
	- target-os: Linux
	runs-on: ubuntu-latest
	shell: bash
	setup-script: \|
	# Set clang as cc
	sudo update-alternatives --install /usr/bin/cc cc $(which clang) 100
	sudo update-alternatives --install /usr/bin/c++ c++ $(which clang++) 100
	sudo update-alternatives --set cc $(which clang)
	sudo update-alternatives --set c++ $(which clang++)

	sudo apt install vulkan-tools vulkan-validationlayers-dev libvulkan-dev -y
	sudo apt install zstd ninja-build ccache -y
	echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
	cmake-init-script: \|
	cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE

	- target-os: Android
	runs-on: ubuntu-latest
	shell: bash
	env:
	CC: clang
	setup-script: \|
	# Set clang as cc
	sudo update-alternatives --install /usr/bin/cc cc $(which clang) 100
	sudo update-alternatives --install /usr/bin/c++ c++ $(which clang++) 100
	sudo update-alternatives --set cc $(which clang)
	sudo update-alternatives --set c++ $(which clang++)

	echo "ANDROID_NDK=$ANDROID_NDK_HOME" >> $GITHUB_ENV
	echo "ANDROID_NDK_HOME=$ANDROID_NDK_HOME" >> $GITHUB_ENV

	sudo apt install zstd ninja-build ccache -y
	echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV

	rustup target add aarch64-linux-android
	arch: \|
	arm64-v8a
	x86_64
	x86
	armeabi-v7a
	cmake-init-script: \|
	cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DANDROID_ABI="arm64-v8a" -DCMAKE_ANDROID_NDK=$ANDROID_NDK -DANDROID_TOOLCHAIN=clang

	- target-os: OSX
	runs-on: macos-latest
	shell: bash
	setup-script: \|
	brew install zstd ninja md5sha1sum ccache
	echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV

	cargo install cbindgen

	rustup target add x86_64-apple-darwin
	cmake-init-script: \|
	cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPLATFORM=MAC

	- target-os: iOS
	runs-on: macos-latest
	shell: bash
	setup-script: \|
	brew install zstd ninja md5sha1sum ccache
	echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV

	cargo install cbindgen

	rustup toolchain install nightly-aarch64-apple-darwin
	rustup component add rust-src --toolchain nightly-aarch64-apple-darwin
	rustup target add aarch64-apple-ios

	cmake-init-script: \|
	cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DiOS_ABI=arm64-v8a -DENABLE_ARC=0 -DENABLE_BITCODE=0 -DENABLE_VISIBILITY=1 -DPLATFORM=OS64

	- target-os: Windows
	runs-on: windows-2022
	shell: bash
	setup-script: \|
	choco install ccache -A
	echo "C:\Strawberry\perl\bin" >> $GITHUB_PATH
	echo "C:\Perl64\bin" >> $GITHUB_PATH
	cmake-init-script: \|
	cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE

	target-save-data:
	- target-os: $TARGET_OS
	runs-on: $TARGET_RUNS_ON
	shell: bash
	upload-release-json: \|
	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	retValue=1
	numberOfTries=1
	until [[ $retValue -eq 0 ]]
	do
	if [[ $numberOfTries -gt 5 ]]; then
	break
	fi

	echo "#${numberOfTries} Uploading release.json to github release ${GITHUB_RELEASE_NAME}"
	gh release upload --clobber ${GITHUB_RELEASE_NAME} release.json
	retValue=$?

	if [[ $retValue -ne 0 ]]; then
	echo "gh release upload failed, sleep 2 sec before next try"
	sleep 2
	fi

	((numberOfTries++))
	done

	if [[ $retValue -ne 0 ]]; then
	echo "Failed to upload release.json to github release ${GITHUB_RELEASE_NAME}"
	exit 1
	fi

	release-target: \|
	# Check if release already defined e.g. Linux-master-release
	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}
	gh release view ${GITHUB_RELEASE_NAME}
	if [ $? -ne 0 ]
	then
	echo "Creating github release with tag: ${GITHUB_RELEASE_NAME}"

	RELEASE_TYPE="--latest"
	if [[ "${BUILD_TYPE}" != "Release" ]];
	then
	RELEASE_TYPE="--prerelease"
	fi

	# Create release on github
	gh release create ${GITHUB_RELEASE_NAME} -n "${BRANCH} branch" --target ${BRANCH} ${RELEASE_TYPE} -t "${TARGET_OS} ${BRANCH} branch ${BUILD_TYPE} build"
	if [ $? -ne 0 ]
	then
	echo "Failed to create github release with tag: ${GITHUB_RELEASE_NAME}"
	exit 1
	fi
	fi

	COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"
	echo "Compressing ${TARGET_OS} version ${TARGET_SHA} of ${TARGET_NAME} ${BUILD_TYPE} mode (target build folder: ${TARGET_DIRECTORY})"
	tar --exclude="$TARGET_NAME/src" --exclude="$TARGET_NAME/tmp" -zcf ${COMPRESSED_TARGET} ${TARGET_DIRECTORY}

	retValue=1
	numberOfTries=1
	until [[ $retValue -eq 0 ]]
	do
	if [[ $numberOfTries -gt 5 ]]; then
	break
	fi

	echo "#${numberOfTries} Uploading target ${COMPRESSED_TARGET} to github release ${GITHUB_RELEASE_NAME}"
	gh release upload --clobber ${GITHUB_RELEASE_NAME} ${COMPRESSED_TARGET}
	retValue=$?

	if [[ $retValue -ne 0 ]]; then
	echo "gh release upload failed, sleep 2 sec before next try"
	sleep 2
	fi

	((numberOfTries++))
	done

	if [[ $retValue -ne 0 ]]; then
	echo "Failed to upload target ${COMPRESSED_TARGET} to github release ${GITHUB_RELEASE_NAME}"
	exit 1
	fi

	runs-on: ${{ matrix.target-build-data.runs-on }}
	outputs:
	TARGET_BUILT: ${{ steps.cache-library-directory.outputs.TARGET_BUILT }}
	steps:
	- name: Create library keys and paths to cache/build
	id: create-lib-cache-keys
	shell: bash
	run: \|
	# build array of library names
	IFS=$'\n' read -d '\|' -ra TARGET_REPO_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_REPO_NAMES }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_DIRECTORIES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_DIRECTORIES }}\|')"
	LIB_CACHE_NAME_KEYS="[ "
	LIB_CACHE_PATHS="[ "
	first=true
	for i in "${!TARGET_REPO_NAMES[@]}"; do
	if [[ "$first" != "true" ]]; then
	LIB_CACHE_NAME_KEYS+=", "
	LIB_CACHE_PATHS+=", "
	fi
	first=false
	LIB_CACHE_NAME_KEYS+="\"l/${{ matrix.target-os }}/${{ matrix.build-type }}/${TARGET_REPO_NAMES[i]}-${TARGET_SHAS[i]}\""
	LIB_CACHE_PATHS+="[\"./build/${{ matrix.target-os }}/${{ matrix.build-type }}/${TARGET_DIRECTORIES[i]}/**\"]"
	done

	LIB_CACHE_NAME_KEYS+=" ]"
	echo "LIB_CACHE_NAME_KEYS=$LIB_CACHE_NAME_KEYS" >> $GITHUB_OUTPUT

	LIB_CACHE_PATHS+=" ]"
	echo "LIB_CACHE_PATHS=$LIB_CACHE_PATHS" >> $GITHUB_OUTPUT

	- name: Remove old libraries that are cached.
	run: \|
	echo "We really should remove old library caches here"

	- name: Install configuration
	uses: ChristopherHX/conditional@main
	with:
	if: ${{ matrix.target-build-data.install-uses != null }}
	step: \|
	uses: ${{ matrix.target-build-data.install-uses.uses \|\| '' }}
	with: ${{ (matrix.target-build-data.install-uses.with && toJSON(matrix.target-build-data.install-uses.with)) \|\| '{}' }}

	- name: Cache Third Party Directory
	uses: GeniusVentures/cache-multi/[email protected]
	id: cache-tp-source-directory
	with:
	key: ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }}
	enableCrossOsArchive: true
	path: \|
	!.git/**
	./**

	- if: (steps.cache-tp-source-directory.outputs.cache-hit != 'true')
	run: \|
	echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
	exit 1

	- name: Cache Submodule source files without .git
	id: cache-source-directory
	uses: GeniusVentures/cache-multi/[email protected]
	with:
	multi-keys: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_NAME_KEYS }}
	enableCrossOsArchive: true
	paths: ${{ needs.setup-src-cache.outputs.CACHE_MULTI_PARAMETER_PATHS }}

	- name: Run setup script for target OS
	shell: bash
	run: \|
	${{ matrix.target-build-data.setup-script }}

	- name: Configure CMake
	shell: bash
	run: \|
	TARGET_OS="${{ matrix.target-build-data.target-os }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	WORKSPACE=${{ github.workspace }}
	cd build/$TARGET_OS
	${{ matrix.target-build-data.cmake-init-script }}

	- name: Load built library caches if any
	id: cache-library-directory
	shell: bash
	run: \|
	TARGET_OS="${{ matrix.target-build-data.target-os }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	BRANCH=${{ github.ref_name }}
	IFS=$'\n' read -d '\|' -ra TARGET_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_NAMES }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}\|')"

	set +e

	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	# check if release.json file already available
	jsonStr="[]"
	jsonFound=`gh release view ${GITHUB_RELEASE_NAME} --json assets --jq "any(.assets[]; .name == \"release.json\")"`
	if [ $? -eq 0 ]
	then
	echo "Downloading release.json file"
	gh release download ${GITHUB_RELEASE_NAME} --pattern "release.json" --clobber
	if [ $? -eq 0 ]
	then
	echo "Loading release.json file"
	jsonStr=`cat release.json`
	fi
	fi

	set -e

	targetBuiltList=""
	for i in "${!TARGET_NAMES[@]}"; do
	TARGET_NAME="${TARGET_NAMES[i]}"
	TARGET_SHA="${TARGET_SHAS[i]}"
	COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"

	targetBuilt=false
	shaFound=`jq ".[] \| select(.name == \"${TARGET_NAME}\") \| .sha" <<< ${jsonStr}`
	if [[ "${shaFound}" == \"${TARGET_SHA}\" ]];
	then
	targetBuilt=true
	fi

	echo "Check if ${COMPRESSED_TARGET} found on ${GITHUB_RELEASE_NAME}: ${targetBuilt}"
	targetBuiltList+="${targetBuilt}\n"
	done

	echo "TARGET_BUILT=$targetBuiltList" >> $GITHUB_OUTPUT

	- name: Install Rust target for wasm
	run: rustup target add wasm32-unknown-emscripten

	# Workaround GitHub shell
	- name: Build OpenSSL
	if: ${{ matrix.target-build-data.target-os == 'Windows' }}
	shell: cmd
	run: \|
	set TARGET_OS=${{ matrix.target-build-data.target-os }}
	set TARGET_RUNS_ON=${{ matrix.target-build-data.runs-on }}
	set BUILD_TYPE=${{ matrix.build-type }}
	cmake --build "%GITHUB_WORKSPACE%\build\%TARGET_OS%\%BUILD_TYPE%" --target openssl --config %BUILD_TYPE% -j

	- name: Build any updated libraries
	id: build-updated-libraries
	shell: bash
	run: \|
	TARGET_OS="${{ matrix.target-build-data.target-os }}"
	TARGET_RUNS_ON="${{ matrix.target-build-data.runs-on }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	BRANCH=${{ github.ref_name }}
	WORKSPACE=${{ github.workspace }}

	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	IFS=$'\n' read -d '\|' -ra TARGET_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_NAMES }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_BUILT <<< "$(echo -ne '${{ steps.cache-library-directory.outputs.TARGET_BUILT }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_DIRECTORIES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_DIRECTORIES }}\|')"

	cd "build/${TARGET_OS}/${BUILD_TYPE}"

	jsonStr="[]"
	declare -a listOfTargetNamesToBuild
	declare -a listOfTargetSHAsToBuild
	declare -a listOfTargetDirToBuild

	for i in "${!TARGET_NAMES[@]}"; do
	TARGET_NAME="${TARGET_NAMES[i]}"
	TARGET_SHA="${TARGET_SHAS[i]}"
	TARGET_DIRECTORY="${TARGET_DIRECTORIES[i]}"
	COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"

	echo "${TARGET_NAME} release found: ${TARGET_BUILT[i]} (Target direcory: $TARGET_DIRECTORY)"

	if [ "${TARGET_BUILT[i]}" == "true" ]; then
	echo "Downloading cached target: $TARGET_NAME, sha: $TARGET_SHA"
	gh release download ${GITHUB_RELEASE_NAME} --pattern "${COMPRESSED_TARGET}" --clobber
	if [ $? -eq 0 ]
	then
	echo "Extracting cached target: ${COMPRESSED_TARGET}"
	tar -xf ${COMPRESSED_TARGET}
	FOLDER_HASH=$(find ./${TARGET_DIRECTORY} -xdev -type f -exec cat {} + \| md5sum \| awk '{print $1}')

	echo "Cached target folder hash: ${FOLDER_HASH}"
	jsonStrHash=$(jq -n \
	--arg name "$TARGET_NAME" \
	--arg sha "$TARGET_SHA" \
	--arg binFolderName "$TARGET_DIRECTORY" \
	--arg binFolderHash "$FOLDER_HASH" \
	'{name: $name, sha: $sha, binFolderName: $binFolderName, binFolderHash: $binFolderHash}')

	jsonStr=$(jq -c --argjson obj "$jsonStrHash" '. + [$obj]' <<< "$jsonStr")

	echo "Target extracted to `pwd`/${TARGET_DIRECTORY}"
	echo "Contents of target folder `pwd`/${TARGET_DIRECTORY}"
	ls -la ./${TARGET_DIRECTORY}
	else
	echo "Failed to download cached target: ${COMPRESSED_TARGET} - need to rebuild it"
	listOfTargetNamesToBuild+=("${TARGET_NAME}")
	listOfTargetSHAsToBuild+=("${TARGET_SHA}")
	listOfTargetDirToBuild+=("${TARGET_DIRECTORY}")
	fi
	else
	echo "Cached target not found: ${COMPRESSED_TARGET} - need to rebuild it"
	listOfTargetNamesToBuild+=("${TARGET_NAME}")
	listOfTargetSHAsToBuild+=("${TARGET_SHA}")
	listOfTargetDirToBuild+=("${TARGET_DIRECTORY}")
	fi
	done

	# Print hash JSON file
	jq <<< $jsonStr

	# save hash JSON file
	jq <<< $jsonStr > hash.json

	# check if release.json file already available
	releaseJsonStr="[]"

	set +e

	jsonFound=`gh release view ${GITHUB_RELEASE_NAME} --json assets --jq "any(.assets[]; .name == \"release.json\")"`
	if [ $? -eq 0 ]
	then
	echo "Downloading release.json file"
	gh release download ${GITHUB_RELEASE_NAME} --pattern "release.json" --clobber
	if [ $? -eq 0 ]
	then
	echo "Loading release.json file"
	releaseJsonStr=`cat release.json`
	fi
	fi

	set -e

	for i in "${!listOfTargetNamesToBuild[@]}"; do
	TARGET_NAME="${listOfTargetNamesToBuild[i]}"
	TARGET_SHA="${listOfTargetSHAsToBuild[i]}"
	TARGET_DIRECTORY="${listOfTargetDirToBuild[i]}"
	COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"
	COMPRESSED_TARGET_SRC="${TARGET_NAME}-src.tar.gz"

	echo "Compiling $TARGET_OS version $TARGET_SHA of $TARGET_NAME $BUILD_TYPE mode"
	cmake --build . --target $TARGET_NAME --config $BUILD_TYPE -j

	echo "Starting to upload target: ${COMPRESSED_TARGET}"
	# Upload released target
	${{ matrix.target-save-data.release-target }}

	RELEASE_ASSETS=`gh release view ${GITHUB_RELEASE_NAME} --json assets`

	# Updating release.json
	# Get source url
	TARGET_SOURCE=`jq ".assets[] \| select(.name == \"${COMPRESSED_TARGET_SRC}\") \| .url" <<< ${RELEASE_ASSETS}`
	# Get target url
	TARGET_BINARY=`jq ".assets[] \| select(.name == \"${COMPRESSED_TARGET}\") \| .url" <<< ${RELEASE_ASSETS}`

	releaseFound=$(jq ".[] \| select(.name == \"${TARGET_NAME}\")" <<<${releaseJsonStr})
	if [ ${#releaseFound} -gt 0 ]
	then
	echo "Updating release.json file with target ${TARGET_NAME}"
	releaseJsonStr=$(jq --arg name "${TARGET_NAME}" \
	--arg sha "${TARGET_SHA}" \
	--arg binFolderName "${TARGET_DIRECTORY}" \
	--arg source "${TARGET_SOURCE}" \
	--arg binary "${TARGET_BINARY}" \
	'(.[] \| select(.name == $name)) \|= (.sha = $sha \| .binFolderName = $binFolderName \| .source = $source \| .binary = $binary)' <<<"${releaseJsonStr}")

	else
	echo "Adding target ${TARGET_NAME} to release.json file"
	jsonStrTarget=$(jq -n \
	--arg name "${TARGET_NAME}" \
	--arg sha "${TARGET_SHA}" \
	--arg binFolderName "${TARGET_DIRECTORY}" \
	--arg source "${TARGET_SOURCE}" \
	--arg binary "${TARGET_BINARY}" \
	'{name: $name, sha: $sha, binFolderName: $binFolderName, source: $source, binary: $binary}')

	releaseJsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$releaseJsonStr")
	fi

	# Upload JSON file release
	jq <<< $releaseJsonStr > release.json

	echo "Uploading JSON file release"
	${{ matrix.target-save-data.upload-release-json }}

	done

	- name: Upload JSON file release on build failure
	if: ${{ failure() && steps.build-updated-libraries.conclusion == 'failure' }}
	shell: bash
	run: \|
	TARGET_OS="${{ matrix.target-build-data.target-os }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	BRANCH=${{ github.ref_name }}

	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	retValue=1
	numberOfTries=1
	until [[ $retValue -eq 0 ]]
	do
	if [[ $numberOfTries -gt 5 ]]; then
	break
	fi

	echo "#${numberOfTries} Uploading release.json to github release ${GITHUB_RELEASE_NAME}"
	gh release upload --clobber ${GITHUB_RELEASE_NAME} release.json
	retValue=$?

	if [[ $retValue -ne 0 ]]; then
	echo "gh release upload failed, sleep 2 sec before next try"
	sleep 2
	fi

	((numberOfTries++))
	done

	- name: Save target OS SDK as github release
	shell: bash
	run: \|
	TARGET_OS="${{ matrix.target-build-data.target-os }}"
	TARGET_RUNS_ON="${{ matrix.target-build-data.runs-on }}"
	BUILD_TYPE="${{ matrix.build-type }}"
	BRANCH=${{ github.ref_name }}
	IFS=$'\n' read -d '\|' -ra TARGET_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_NAMES }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_BUILT <<< "$(echo -ne '${{ steps.cache-library-directory.outputs.TARGET_BUILT }}\|')"
	IFS=$'\n' read -d '\|' -ra TARGET_DIRECTORIES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_DIRECTORIES }}\|')"

	GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}

	RELEASE_ASSETS=`gh release view ${GITHUB_RELEASE_NAME} --json assets`

	jsonStr="[]"
	cd "build/${TARGET_OS}/${BUILD_TYPE}"
	for i in "${!TARGET_NAMES[@]}"; do
	TARGET_NAME="${TARGET_NAMES[i]}"
	TARGET_SHA="${TARGET_SHAS[i]}"
	TARGET_DIRECTORY="${TARGET_DIRECTORIES[i]}"

	COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"
	COMPRESSED_TARGET_SRC="${TARGET_NAME}-src.tar.gz"
	echo "${TARGET_NAME} release found: ${TARGET_BUILT[i]}"

	if [ "${TARGET_BUILT[i]}" == "true" ]; then
	# Compare uncompressed original hash of the binary folder with current to see if any changes has been made
	originalHash=`jq ".[] \| select(.name == \"${TARGET_NAME}\") \| .binFolderHash" hash.json`
	currentHash=$(find ./${TARGET_DIRECTORY} -xdev -type f -exec cat {} + \| md5sum \| awk '{print $1}')
	if [[ "${originalHash}" != \"${currentHash}\" ]];
	then
	echo "originalHash: ${originalHash}"
	echo "currentHash: \"${currentHash}\""
	echo "Has does not match, uploading already installed modified target: ${TARGET_NAME}, sha: ${TARGET_SHA}"
	${{ matrix.target-save-data.release-target }}
	else
	echo "Target ${TARGET_NAME} binary folder did not change, no need to update release"
	fi
	fi

	# Update JSON file
	# Get source url
	TARGET_SOURCE=`jq ".assets[] \| select(.name == \"${COMPRESSED_TARGET_SRC}\") \| .url" <<< ${RELEASE_ASSETS}`
	# Get target url
	TARGET_BINARY=`jq ".assets[] \| select(.name == \"${COMPRESSED_TARGET}\") \| .url" <<< ${RELEASE_ASSETS}`
	jsonStrTarget=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"source\" : ${TARGET_SOURCE}, \"binary\" : ${TARGET_BINARY} }" <<< "{}"`
	jsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$jsonStr")
	done

	# Print final JSON file
	jq <<< $jsonStr

	# Upload JSON file release
	jq <<< $jsonStr > release.json

	${{ matrix.target-save-data.upload-release-json }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Multiplatform build and upload #413

Workflow file

Multiplatform build and upload #413

Jobs

Run details

Workflow file for this run