Skip to content

Commit

Permalink
Fixed build targets build
Browse files Browse the repository at this point in the history
  • Loading branch information
@EduMenges
EduMenges committed Jul 17, 2024
1 parent 3cd4670 commit cfeb517
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 64 deletions.
115 changes: 58 additions & 57 deletions .github/workflows/build-targets.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,6 @@ jobs:
foundThirdPartyCurrent="false"
fi
# Allows the script to continue even if the cache deletion commands fail
set +e
# Set the field separator to new line
IFS=$'\n'
for oldCache in $oldCaches; do
Expand Down Expand Up @@ -123,9 +120,6 @@ jobs:
done
matrixJson+=" ]"
# Setting this to not fail the workflow while deleting cache keys.
set +e
# Delete old caches
for oldCache in $oldCaches; do
echo "Deleting old cache $oldCache"
Expand Down Expand Up @@ -198,7 +192,7 @@ jobs:
- name: Git clone submodule source code
run: |
echo "Cloning source for ${{ matrix.name }}"
git submodule update --init --recursive --recommend-shallow -- ${{ matrix.name }}
git submodule update --init --recommend-shallow --recursive -- ${{ matrix.name }}
shell: bash
working-directory: ${{github.workspace}}

Expand Down Expand Up @@ -257,28 +251,29 @@ jobs:
cmake -S build/Linux -B build/Linux/Release -DCMAKE_BUILD_TYPE=Release
cd build/Linux/Release/CMakeFiles
declare -A cMakeBuildDir
declare -A cmakeBuildDir
while IFS=',' read -r cmakeName buildDir; do
cMakeBuildDir["$cmakeName"]="$buildDir"
cmakeBuildDir["$cmakeName"]="$buildDir"
done < <(grep -m1 -R --include "build.make" -e ".*-complete:.*-build" * |
sed -E -n 's/^(.*)\.dir.*-complete: ([^/]*?)\/src.*/\1,\2/p')
cd ..
declare -A cMakeNamesRepos
for key in "${!cMakeBuildDir[@]}"; do
sourceDir=$(find . -type f -name "${key}-source_dirinfo.txt" -exec sed -n 's/^source_dir=\(.*\)\/\(.*\)/\2/p' {} \;)
cMakeNamesRepos["$key"]=$sourceDir
declare -A cmakeNamesRepos
for key in "${!cmakeBuildDir[@]}"; do
sourceDir=$(find . -type f -name "${key}-source_dirinfo.txt" -exec sed -n 's/^source_dir=${GITHUB_WORKSPACE}\/\(.*\)/\1/p' {} \;)
cmakeNamesRepos["$key"]=$sourceDir
done
declare -A gitLibModules
cd ../../..
while IFS=',' read -r longSHA sourceDir; do
echo $longSha $sourceDir
gitLibModules["$sourceDir"]="$longSHA"
done < <(git ls-tree HEAD |
sed -E -n 's/^(160000 commit|040000 tree) ([a-f0-9]+)[ \t]+([^ ]+)$/\2,\3/p')
# Initialize the variables
cMakeNames=""
cmakeNames=""
buildDirectoryNames=""
repoNames=""
shortSHAs=""
Expand All @@ -290,18 +285,18 @@ jobs:
# Print header
printf "%-${name_width}s | %-${build_dir_width}s | %-${source_dir_width}s | %-${sha_width}s\n" "Name" "Build dir" "Source dir" "SHA"
for key in "${!cMakeBuildDir[@]}"; do
buildDir="${cMakeBuildDir[$key]}"
sourceDir="${cMakeNamesRepos[$key]}"
for key in "${!cmakeBuildDir[@]}"; do
buildDir="${cmakeBuildDir[$key]}"
sourceDir="${cmakeNamesRepos[$key]}"
shortSHA="${gitLibModules[$sourceDir]:0:7}"
cMakeNames+="${key}\n"
cmakeNames+="${key}\n"
buildDirectoryNames+="${buildDir}\n"
repoNames+="${sourceDir}\n"
shortSHAs+="${shortSHA}\n"
printf "%-${name_width}s | %-${build_dir_width}s | %-${source_dir_width}s | %s\n" "${key}" "${buildDir}" "${sourceDir}" "${shortSHA}"
done
echo "TARGET_NAMES=$cMakeNames" >> $GITHUB_OUTPUT
echo "TARGET_NAMES=$cmakeNames" >> $GITHUB_OUTPUT
echo "TARGET_SHAS=$shortSHAs" >> $GITHUB_OUTPUT
echo "TARGET_DIRECTORIES=$buildDirectoryNames" >> $GITHUB_OUTPUT
echo "TARGET_REPO_NAMES=$repoNames" >> $GITHUB_OUTPUT
Expand Down Expand Up @@ -356,7 +351,7 @@ jobs:
echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
exit 1
- name: Cache Submodule Source Files without .git
- name: Cache Submodule source files without .git
id: cache-source-directory
uses: GeniusVentures/cache-multi/[email protected]
with:
Expand All @@ -367,11 +362,13 @@ jobs:
- name: Creating github release tag
id: create-github-release-tag
run: |
set +e
echo "Check if github release tag ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} available."
gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}
set +e
gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}
releaseFound=$?
set -e
if [[ $releaseFound -ne 0 ]];
then
echo "gh release view return value: ${releaseFound}"
Expand All @@ -393,15 +390,12 @@ jobs:
releaseCreated=$?
if [[ $releaseCreated -ne 0 ]];
then
set -e
echo "gh release create return value: ${releaseCreated}"
echo "Failed to create github release with tag: ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}"
exit 1
fi
fi
set -e
- name: Compress and upload targets source folder
id: compress-targets-source
working-directory: ${{github.workspace}}
Expand All @@ -411,19 +405,20 @@ jobs:
IFS=$'\n' read -d '|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}|')"
IFS=$'\n' read -d '|' -ra TARGET_REPO_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_REPO_NAMES }}|')"
set +e
# check if release.json file already available
# Check if release.json file already available
jsonStr="[]"
jsonFound=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets --jq "any(.assets[]; .name == \"release.json\")"`
if [ $? -eq 0 ]
then
echo "Downloading release.json file"
set +e
gh release download ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --pattern "release.json" --clobber
if [ $? -eq 0 ]
then
echo "Loading release.json file"
jsonStr=`cat release.json`
fi
set -e
fi
RELEASE_ASSETS=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets`
Expand Down Expand Up @@ -455,7 +450,7 @@ jobs:
TARGET_SHA="${TARGET_SHAS[i]}"
COMPRESSED_TARGET_SRC="${TARGET_NAME}-src.tar.gz"
shaFound=`jq ". | select(.name == \"${TARGET_NAME}\") | .sha" <<< ${jsonStr}`
shaFound=$(jq ".[] | select(.name == \"${TARGET_NAME}\") | first | .sha" <<<${jsonStr})
COMPRESSED_TARGET_SRC_URL=`jq ".assets[] | select(.name == \"${COMPRESSED_TARGET_SRC}\") | .url" <<< ${RELEASE_ASSETS}`
if [ -z "$COMPRESSED_TARGET_SRC_URL" ] || [[ "${shaFound}" != \"${TARGET_SHA}\" ]];
then
Expand All @@ -465,7 +460,6 @@ jobs:
gh release upload --clobber ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} ${COMPRESSED_TARGET_SRC}
fi
done
set -e
build-targets-matrix:
needs:
Expand All @@ -487,7 +481,8 @@ jobs:
shell: bash
init-script:
setup-script: |
sudo apt install zstd -y
sudo apt install ninja-build zstd -y
echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
cmake-init-script: |
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
Expand All @@ -503,7 +498,8 @@ jobs:
git config --global core.eol lf
echo "ANDROID_NDK=$ANDROID_NDK_HOME" >> $GITHUB_ENV
echo "ANDROID_NDK_HOME=$ANDROID_NDK_HOME" >> $GITHUB_ENV
sudo apt install zstd -y
sudo apt install zstd ninja-build -y
echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
arch: |
arm64-v8a
x86_64
Expand All @@ -520,11 +516,12 @@ jobs:
setup-script: |
find /Applications/Xcode* -maxdepth 0 -ls
sudo xcode-select --switch "$(find /Applications -mindepth 1 -maxdepth 1 ! -type l | grep "Xcode_[\.0-9]*app" | sort -V | tail -1)/Contents/Developer"
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
brew install zstd
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
brew install zstd ninja
echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
cargo install cbindgen
cmake-init-script: |
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPLATFORM=MAC
# --------------------------------------------------------------------------
- target-os: iOS
Expand All @@ -538,11 +535,12 @@ jobs:
find /Applications/Xcode* -maxdepth 0 -ls
sudo xcode-select --switch "$(find /Applications -mindepth 1 -maxdepth 1 ! -type l | grep "Xcode_*[\.0-9]*app" | sort -V | tail -1)/Contents/Developer"
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
brew install zstd
brew install zstd ninja
echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV
cargo install cbindgen
cmake-init-script: |
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DiOS_ABI=arm64-v8a -DENABLE_ARC=0 -DENABLE_BITCODE=0 -DENABLE_VISIBILITY=1 -DCMAKE_TOOLCHAIN_FILE=${WORKSPACE}/build/iOS/iOS.cmake
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DiOS_ABI=arm64-v8a -DENABLE_ARC=0 -DENABLE_BITCODE=0 -DENABLE_VISIBILITY=1 -DPLATFORM=OS64
# --------------------------------------------------------------------------
- target-os: Windows
Expand All @@ -558,7 +556,7 @@ jobs:
echo "Adding GNU tar to PATH"
echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
cmake-init-script: |
cmake -S .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE
cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE
target-save-data:
- target-os: $TARGET_OS
Expand Down Expand Up @@ -588,7 +586,6 @@ jobs:
done
if [[ $retValue -ne 0 ]]; then
set -e
echo "Failed to upload release.json to github release ${GITHUB_RELEASE_NAME}"
exit 1
fi
Expand All @@ -611,15 +608,14 @@ jobs:
gh release create ${GITHUB_RELEASE_NAME} -n "${BRANCH} branch" --target ${BRANCH} ${RELEASE_TYPE} -t "${TARGET_OS} ${BRANCH} branch ${BUILD_TYPE} build"
if [ $? -ne 0 ]
then
set -e
echo "Failed to create github release with tag: ${GITHUB_RELEASE_NAME}"
exit 1
fi
fi
COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"
echo "Compressing ${TARGET_OS} version ${TARGET_SHA} of ${TARGET_NAME} ${BUILD_TYPE} mode (target build folder: ${TARGET_DIRECTORY})"
tar --exclude="src/" --exclude"tmp/" --exclude="build/" -zcf ${COMPRESSED_TARGET} ${TARGET_DIRECTORY}
tar --exclude="src/" --exclude="tmp/" -zcf ${COMPRESSED_TARGET} ${TARGET_DIRECTORY}
retValue=1
numberOfTries=1
Expand All @@ -642,7 +638,6 @@ jobs:
done
if [[ $retValue -ne 0 ]]; then
set -e
echo "Failed to upload target ${COMPRESSED_TARGET} to github release ${GITHUB_RELEASE_NAME}"
exit 1
fi
Expand Down Expand Up @@ -705,7 +700,7 @@ jobs:
echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!"
exit 1
- name: Cache Submodule Source Files without .git
- name: Cache Submodule source files without .git
id: cache-source-directory
uses: GeniusVentures/cache-multi/[email protected]
with:
Expand All @@ -718,9 +713,7 @@ jobs:
run: |
echo "Running init script"
set +e
${{ matrix.target-build-data.init-script }}
set -e
echo "Running setup script"
${{ matrix.target-build-data.setup-script }}
Expand All @@ -744,8 +737,6 @@ jobs:
IFS=$'\n' read -d '|' -ra TARGET_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_NAMES }}|')"
IFS=$'\n' read -d '|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}|')"
set +e
GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}
# check if release.json file already available
Expand Down Expand Up @@ -800,18 +791,19 @@ jobs:
cd "build/${TARGET_OS}/${BUILD_TYPE}"
set +e
jsonStr=""
declare -a listOfTargetNamesToBuild
declare -a listOfTargetSHAsToBuild
declare -a listOfTargetDirToBuild
for i in "${!TARGET_NAMES[@]}"; do
TARGET_NAME="${TARGET_NAMES[i]}"
TARGET_SHA="${TARGET_SHAS[i]}"
TARGET_DIRECTORY="${TARGET_DIRECTORIES[i]}"
COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz"
echo "${TARGET_NAME} release found: ${TARGET_BUILT[i]} (Target direcory: $TARGET_DIRECTORY)"
if [ "${TARGET_BUILT[i]}" == "true" ]; then
echo "Downloading cached target: $TARGET_NAME, sha: $TARGET_SHA"
gh release download ${GITHUB_RELEASE_NAME} --pattern "${COMPRESSED_TARGET}" --clobber
Expand All @@ -821,8 +813,14 @@ jobs:
tar -xf ${COMPRESSED_TARGET}
FOLDER_HASH=`find ./${TARGET_DIRECTORY} -xdev -type f -print0 | LC_COLLATE=C sort -z | xargs -0 cat | md5sum | awk '{print $1}'`
echo "Cached target folder hash: ${FOLDER_HASH}"
jsonStrHash=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"binFolderHash\" : \"${FOLDER_HASH}\" }" <<< "{}"`
jsonStr+=`echo ${jsonStrHash}`
jsonStrHash=$(jq -n \
--arg name "$TARGET_NAME" \
--arg sha "$TARGET_SHA" \
--arg binFolderName "$TARGET_DIRECTORY" \
--arg binFolderHash "$FOLDER_HASH" \
'{name: $name, sha: $sha, binFolderName: $binFolderName, binFolderHash: $binFolderHash}')
jsonStr=$(jq -c --argjson obj "$jsonStrHash" '. + [$obj]' <<< "$jsonStr")
echo "Target extracted to `pwd`/${TARGET_DIRECTORY}"
echo "Contents of target folder `pwd`/${TARGET_DIRECTORY}"
Expand Down Expand Up @@ -870,7 +868,6 @@ jobs:
echo "Compiling $TARGET_OS version $TARGET_SHA of $TARGET_NAME $BUILD_TYPE mode"
cmake --build . --target $TARGET_NAME --config $BUILD_TYPE -j
cmake --install $TARGET_NAME
echo "Starting to upload target: ${COMPRESSED_TARGET}"
# Upload released target
Expand All @@ -894,8 +891,15 @@ jobs:
releaseJsonStr=`jq "(. | select(.name == \"${TARGET_NAME}\")).binary |= \"${TARGET_BINARY}\" " <<< ${releaseJsonStr}`
else
echo "Adding target ${TARGET_NAME} to release.json file"
jsonStrTarget=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"source\" : ${TARGET_SOURCE}, \"binary\" : ${TARGET_BINARY} }" <<< "{}"`
releaseJsonStr+=`echo ${jsonStrTarget}`
jsonStrTarget=$(jq -n \
--arg name "${TARGET_NAME}" \
--arg sha "${TARGET_SHA}" \
--arg binFolderName "${TARGET_DIRECTORY}" \
--arg source "${TARGET_SOURCE}" \
--arg binary "${TARGET_BINARY}" \
'{name: $name, sha: $sha, binFolderName: $binFolderName, source: $source, binary: $binary}')
releaseJsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$releaseJsonStr")
fi
# Upload JSON file release
Expand Down Expand Up @@ -947,9 +951,6 @@ jobs:
IFS=$'\n' read -d '|' -ra TARGET_BUILT <<< "$(echo -ne '${{ steps.cache-library-directory.outputs.TARGET_BUILT }}|')"
IFS=$'\n' read -d '|' -ra TARGET_DIRECTORIES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_DIRECTORIES }}|')"
# Setting this to not fail the workflow while checking release tag
set +e
GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE}
RELEASE_ASSETS=`gh release view ${GITHUB_RELEASE_NAME} --json assets`
Expand Down Expand Up @@ -985,7 +986,7 @@ jobs:
# Get target url
TARGET_BINARY=`jq ".assets[] | select(.name == \"${COMPRESSED_TARGET}\") | .url" <<< ${RELEASE_ASSETS}`
jsonStrTarget=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"source\" : ${TARGET_SOURCE}, \"binary\" : ${TARGET_BINARY} }" <<< "{}"`
jsonStr+=`echo ${jsonStrTarget}`
jsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$jsonStr")
done
# Print final JSON file
Expand Down
Loading

0 comments on commit cfeb517

Please sign in to comment.