diff --git a/.github/workflows/build-targets.yml b/.github/workflows/build-targets.yml index 1e5859a..38b1db9 100644 --- a/.github/workflows/build-targets.yml +++ b/.github/workflows/build-targets.yml @@ -58,9 +58,6 @@ jobs: foundThirdPartyCurrent="false" fi - # Allows the script to continue even if the cache deletion commands fail - set +e - # Set the field separator to new line IFS=$'\n' for oldCache in $oldCaches; do @@ -92,7 +89,7 @@ jobs: while read -r mode stage longsha location; do shortSHA="${longsha:0:7}" cacheNames+="s/$BRANCH/$location-$shortSHA\n" - done <<< $(git ls-tree HEAD | grep "160000 commit") + done <<< $(git ls-tree -r HEAD | grep "160000 commit") cacheNames="$(echo -en "$cacheNames" | sort)" cachesList=$(gh api -H "Accept: application/vnd.github+json" --paginate repos/GeniusVentures/thirdparty/actions/caches?key="s/${BRANCH}" | jq -r ".actions_caches[].key" | sort) @@ -101,30 +98,26 @@ jobs: oldCaches="$(comm -13 --output-delimiter="" <(echo -en "$cacheNames") <(echo -en "$notMatched"))" notCached="$(comm -12 --output-delimiter="" <(echo -en "$cacheNames") <(echo -en "$notMatched"))" - # matrixJson = matrix of source code to cache, that isn't already cached - matrixJson="[ " - first=true + # Matrix of source code to cache + cacheMatrix="[]" + # Set the field separator to new line IFS=$'\n' for toCache in $notCached; do len=${#toCache} startPos=$((len - 7)) - shortSHA="${toCache:$startPos:7}" lenBranch=${#BRANCH} - - # - 10 - lenBranch - 1 for s/$BRANCH/ + '-' + shortSHA - name="${toCache:$((2 + lenBranch + 1)):$((len - 10 - lenBranch - 1))}" - if [[ "$first" != "true" ]]; then - matrixJson+=", " - fi - first=false - - matrixJson+="{ \"name\": \"$name\", \"cache-name\": \"$toCache\" }" + matrixItem=$(jq -n \ + --arg cacheName "${toCache}" \ + --arg name "${toCache:$((2 + lenBranch + 1)):$((len - 10 - lenBranch - 1))}" \ + '{name: $name, cacheName: $cacheName}' + ) + + cacheMatrix=$(jq -c --argjson obj "$matrixItem" '. + [$obj]' <<<"$cacheMatrix") done - matrixJson+=" ]" - - # Setting this to not fail the workflow while deleting cache keys. - set +e + + echo "Cache matrix:" + echo $cacheMatrix # Delete old caches for oldCache in $oldCaches; do @@ -132,7 +125,7 @@ jobs: gh actions-cache delete $oldCache -R $REPO -B $BRANCH --confirm done - echo "CACHE_MATRIX=$matrixJson" >> $GITHUB_OUTPUT + echo "CACHE_MATRIX=$cacheMatrix" >> $GITHUB_OUTPUT if [[ "$notCached" != "" ]]; then echo "NEEDS_CACHE=true" >> $GITHUB_OUTPUT else @@ -198,7 +191,7 @@ jobs: - name: Git clone submodule source code run: | echo "Cloning source for ${{ matrix.name }}" - git submodule update --init --recursive --recommend-shallow -- ${{ matrix.name }} + git submodule update --init --recommend-shallow --recursive -- ${{ matrix.name }} shell: bash working-directory: ${{github.workspace}} @@ -206,7 +199,7 @@ jobs: id: cache-source-directory uses: GeniusVentures/cache-multi@v3.3.1 with: - key: ${{ matrix.cache-name }} + key: ${{ matrix.cacheName }} enableCrossOsArchive: true path: | !./${{ matrix.name }}/.git/** @@ -257,28 +250,28 @@ jobs: cmake -S build/Linux -B build/Linux/Release -DCMAKE_BUILD_TYPE=Release cd build/Linux/Release/CMakeFiles - declare -A cMakeBuildDir + declare -A cmakeBuildDir while IFS=',' read -r cmakeName buildDir; do - cMakeBuildDir["$cmakeName"]="$buildDir" + cmakeBuildDir["$cmakeName"]="$buildDir" done < <(grep -m1 -R --include "build.make" -e ".*-complete:.*-build" * | sed -E -n 's/^(.*)\.dir.*-complete: ([^/]*?)\/src.*/\1,\2/p') cd .. - declare -A cMakeNamesRepos - for key in "${!cMakeBuildDir[@]}"; do - sourceDir=$(find . -type f -name "${key}-source_dirinfo.txt" -exec sed -n 's/^source_dir=\(.*\)\/\(.*\)/\2/p' {} \;) - cMakeNamesRepos["$key"]=$sourceDir + declare -A cmakeNamesRepos + for key in "${!cmakeBuildDir[@]}"; do + sourceDir=$(find . -type f -name "${key}-source_dirinfo.txt" -exec sed -n "s|^source_dir=$GITHUB_WORKSPACE\/\(.*\)|\1|p" {} \;) + cmakeNamesRepos["$key"]=$sourceDir + echo "cmakeNameRepos = ${sourceDir}" done declare -A gitLibModules - cd ../../.. + cd $GITHUB_WORKSPACE while IFS=',' read -r longSHA sourceDir; do gitLibModules["$sourceDir"]="$longSHA" - done < <(git ls-tree HEAD | - sed -E -n 's/^(160000 commit|040000 tree) ([a-f0-9]+)[ \t]+([^ ]+)$/\2,\3/p') + done < <(git ls-tree -r HEAD | sed -E -n 's/^.*(commit|tree)\s+([a-f0-9]+)\s+(\S+)/\2,\3/p') # Initialize the variables - cMakeNames="" + cmakeNames="" buildDirectoryNames="" repoNames="" shortSHAs="" @@ -290,18 +283,18 @@ jobs: # Print header printf "%-${name_width}s | %-${build_dir_width}s | %-${source_dir_width}s | %-${sha_width}s\n" "Name" "Build dir" "Source dir" "SHA" - for key in "${!cMakeBuildDir[@]}"; do - buildDir="${cMakeBuildDir[$key]}" - sourceDir="${cMakeNamesRepos[$key]}" + for key in "${!cmakeBuildDir[@]}"; do + buildDir="${cmakeBuildDir[$key]}" + sourceDir="${cmakeNamesRepos[$key]}" shortSHA="${gitLibModules[$sourceDir]:0:7}" - cMakeNames+="${key}\n" + cmakeNames+="${key}\n" buildDirectoryNames+="${buildDir}\n" repoNames+="${sourceDir}\n" shortSHAs+="${shortSHA}\n" printf "%-${name_width}s | %-${build_dir_width}s | %-${source_dir_width}s | %s\n" "${key}" "${buildDir}" "${sourceDir}" "${shortSHA}" done - echo "TARGET_NAMES=$cMakeNames" >> $GITHUB_OUTPUT + echo "TARGET_NAMES=$cmakeNames" >> $GITHUB_OUTPUT echo "TARGET_SHAS=$shortSHAs" >> $GITHUB_OUTPUT echo "TARGET_DIRECTORIES=$buildDirectoryNames" >> $GITHUB_OUTPUT echo "TARGET_REPO_NAMES=$repoNames" >> $GITHUB_OUTPUT @@ -356,7 +349,7 @@ jobs: echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!" exit 1 - - name: Cache Submodule Source Files without .git + - name: Cache Submodule source files without .git id: cache-source-directory uses: GeniusVentures/cache-multi/restoremulti@v3.3.1 with: @@ -367,11 +360,13 @@ jobs: - name: Creating github release tag id: create-github-release-tag run: | - set +e echo "Check if github release tag ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} available." - gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} + set +e + gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} releaseFound=$? + set -e + if [[ $releaseFound -ne 0 ]]; then echo "gh release view return value: ${releaseFound}" @@ -393,15 +388,12 @@ jobs: releaseCreated=$? if [[ $releaseCreated -ne 0 ]]; then - set -e echo "gh release create return value: ${releaseCreated}" echo "Failed to create github release with tag: ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }}" exit 1 fi fi - set -e - - name: Compress and upload targets source folder id: compress-targets-source working-directory: ${{github.workspace}} @@ -411,19 +403,20 @@ jobs: IFS=$'\n' read -d '|' -ra TARGET_SHAS <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_SHAS }}|')" IFS=$'\n' read -d '|' -ra TARGET_REPO_NAMES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_REPO_NAMES }}|')" - set +e - # check if release.json file already available + # Check if release.json file already available jsonStr="[]" jsonFound=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets --jq "any(.assets[]; .name == \"release.json\")"` if [ $? -eq 0 ] then echo "Downloading release.json file" + set +e gh release download ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --pattern "release.json" --clobber if [ $? -eq 0 ] then echo "Loading release.json file" jsonStr=`cat release.json` fi + set -e fi RELEASE_ASSETS=`gh release view ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} --json assets` @@ -455,8 +448,8 @@ jobs: TARGET_SHA="${TARGET_SHAS[i]}" COMPRESSED_TARGET_SRC="${TARGET_NAME}-src.tar.gz" - shaFound=`jq ". | select(.name == \"${TARGET_NAME}\") | .sha" <<< ${jsonStr}` - COMPRESSED_TARGET_SRC_URL=`jq ".assets[] | select(.name == \"${COMPRESSED_TARGET_SRC}\") | .url" <<< ${RELEASE_ASSETS}` + shaFound=$(jq ".[] | select(.name == \"${TARGET_NAME}\") | .sha" <<<${jsonStr}) + COMPRESSED_TARGET_SRC_URL=$(jq '.assets[] | select(.name == "${COMPRESSED_TARGET_SRC}") | .url' <<<${RELEASE_ASSETS}) if [ -z "$COMPRESSED_TARGET_SRC_URL" ] || [[ "${shaFound}" != \"${TARGET_SHA}\" ]]; then echo "Compressing ${TARGET_OS} version ${TARGET_SHA} of ${TARGET_REPO_NAME} source folder" @@ -465,7 +458,6 @@ jobs: gh release upload --clobber ${{ steps.create-src-cache-keys.outputs.GITHUB_RELEASE_NAME }} ${COMPRESSED_TARGET_SRC} fi done - set -e build-targets-matrix: needs: @@ -481,29 +473,40 @@ jobs: matrix: build-type: [Release, Debug] target-build-data: - # -------------------------------------------------------------------------- - target-os: Linux runs-on: ubuntu-latest shell: bash - init-script: setup-script: | - sudo apt install zstd -y + # Set clang as cc + sudo update-alternatives --install /usr/bin/cc cc $(which clang) 100 + sudo update-alternatives --install /usr/bin/c++ c++ $(which clang++) 100 + sudo update-alternatives --set cc $(which clang) + sudo update-alternatives --set c++ $(which clang++) + + sudo apt install ninja-build zstd -y + echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV cmake-init-script: | cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE - # -------------------------------------------------------------------------- - target-os: Android runs-on: ubuntu-latest shell: bash env: CC: clang - init-script: setup-script: | - git config --global core.autocrlf false - git config --global core.eol lf + # Set clang as cc + sudo update-alternatives --install /usr/bin/cc cc $(which clang) 100 + sudo update-alternatives --install /usr/bin/c++ c++ $(which clang++) 100 + sudo update-alternatives --set cc $(which clang) + sudo update-alternatives --set c++ $(which clang++) + echo "ANDROID_NDK=$ANDROID_NDK_HOME" >> $GITHUB_ENV echo "ANDROID_NDK_HOME=$ANDROID_NDK_HOME" >> $GITHUB_ENV - sudo apt install zstd -y + + sudo apt install zstd ninja-build -y + echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV + + rustup target add aarch64-linux-android arch: | arm64-v8a x86_64 @@ -512,53 +515,42 @@ jobs: cmake-init-script: | cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DANDROID_ABI="arm64-v8a" -DCMAKE_ANDROID_NDK=$ANDROID_NDK -DANDROID_TOOLCHAIN=clang - # -------------------------------------------------------------------------- - target-os: OSX runs-on: macos-latest shell: bash - init-script: setup-script: | - find /Applications/Xcode* -maxdepth 0 -ls - sudo xcode-select --switch "$(find /Applications -mindepth 1 -maxdepth 1 ! -type l | grep "Xcode_[\.0-9]*app" | sort -V | tail -1)/Contents/Developer" - sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/* - brew install zstd + brew install zstd ninja md5sha1sum + echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV + cargo install cbindgen + + rustup target add x86_64-apple-darwin cmake-init-script: | - cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE + cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPLATFORM=MAC - # -------------------------------------------------------------------------- - target-os: iOS runs-on: macos-latest shell: bash - init-script: - uses: actions/setup-python@v2 - with: - python-version: "3.5" setup-script: | - find /Applications/Xcode* -maxdepth 0 -ls - sudo xcode-select --switch "$(find /Applications -mindepth 1 -maxdepth 1 ! -type l | grep "Xcode_*[\.0-9]*app" | sort -V | tail -1)/Contents/Developer" - sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/* - brew install zstd + brew install zstd ninja md5sha1sum + echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV + cargo install cbindgen + rustup toolchain install nightly-aarch64-apple-darwin + rustup component add rust-src --toolchain nightly-aarch64-apple-darwin + rustup target add aarch64-apple-ios + cmake-init-script: | - cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DiOS_ABI=arm64-v8a -DENABLE_ARC=0 -DENABLE_BITCODE=0 -DENABLE_VISIBILITY=1 -DCMAKE_TOOLCHAIN_FILE=${WORKSPACE}/build/iOS/iOS.cmake + cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DiOS_ABI=arm64-v8a -DENABLE_ARC=0 -DENABLE_BITCODE=0 -DENABLE_VISIBILITY=1 -DPLATFORM=OS64 - # -------------------------------------------------------------------------- - target-os: Windows runs-on: windows-2022 shell: bash - init-script: - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: amd64 - sdk: 10.0.22621.0 - vsversion: 17 setup-script: | - echo "Adding GNU tar to PATH" - echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + choco install ccache -A cmake-init-script: | - cmake -S .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE + cmake -B $BUILD_TYPE -DCMAKE_BUILD_TYPE=$BUILD_TYPE target-save-data: - target-os: $TARGET_OS @@ -588,7 +580,6 @@ jobs: done if [[ $retValue -ne 0 ]]; then - set -e echo "Failed to upload release.json to github release ${GITHUB_RELEASE_NAME}" exit 1 fi @@ -611,7 +602,6 @@ jobs: gh release create ${GITHUB_RELEASE_NAME} -n "${BRANCH} branch" --target ${BRANCH} ${RELEASE_TYPE} -t "${TARGET_OS} ${BRANCH} branch ${BUILD_TYPE} build" if [ $? -ne 0 ] then - set -e echo "Failed to create github release with tag: ${GITHUB_RELEASE_NAME}" exit 1 fi @@ -619,7 +609,7 @@ jobs: COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz" echo "Compressing ${TARGET_OS} version ${TARGET_SHA} of ${TARGET_NAME} ${BUILD_TYPE} mode (target build folder: ${TARGET_DIRECTORY})" - tar --exclude="src/" --exclude"tmp/" --exclude="build/" -zcf ${COMPRESSED_TARGET} ${TARGET_DIRECTORY} + tar --exclude="src/" --exclude="tmp/" -zcf ${COMPRESSED_TARGET} ${TARGET_DIRECTORY} retValue=1 numberOfTries=1 @@ -642,7 +632,6 @@ jobs: done if [[ $retValue -ne 0 ]]; then - set -e echo "Failed to upload target ${COMPRESSED_TARGET} to github release ${GITHUB_RELEASE_NAME}" exit 1 fi @@ -705,7 +694,7 @@ jobs: echo "Couldn't find cached thirdparty code, ${{ needs.setup-src-cache.outputs.THIRD_PARTY_CACHE_NAME_KEY }} fatal error!" exit 1 - - name: Cache Submodule Source Files without .git + - name: Cache Submodule source files without .git id: cache-source-directory uses: GeniusVentures/cache-multi/restoremulti@v3.3.1 with: @@ -716,13 +705,6 @@ jobs: - name: Run setup script for target OS shell: bash run: | - - echo "Running init script" - set +e - ${{ matrix.target-build-data.init-script }} - set -e - - echo "Running setup script" ${{ matrix.target-build-data.setup-script }} - name: Configure CMake @@ -762,6 +744,8 @@ jobs: fi fi + set -e + targetBuiltList="" for i in "${!TARGET_NAMES[@]}"; do TARGET_NAME="${TARGET_NAMES[i]}" @@ -769,7 +753,7 @@ jobs: COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz" targetBuilt=false - shaFound=`jq ". | select(.name == \"${TARGET_NAME}\") | .sha" <<< ${jsonStr}` + shaFound=`jq ".[] | select(.name == \"${TARGET_NAME}\") | .sha" <<< ${jsonStr}` if [[ "${shaFound}" == \"${TARGET_SHA}\" ]]; then targetBuilt=true @@ -781,6 +765,9 @@ jobs: echo "TARGET_BUILT=$targetBuiltList" >> $GITHUB_OUTPUT + - name: Install Rust target for wasm + run: rustup target add wasm32-unknown-emscripten + - name: Build any updated libraries id: build-updated-libraries shell: bash @@ -800,18 +787,19 @@ jobs: cd "build/${TARGET_OS}/${BUILD_TYPE}" - set +e - - jsonStr="" + jsonStr="[]" declare -a listOfTargetNamesToBuild declare -a listOfTargetSHAsToBuild declare -a listOfTargetDirToBuild + for i in "${!TARGET_NAMES[@]}"; do TARGET_NAME="${TARGET_NAMES[i]}" TARGET_SHA="${TARGET_SHAS[i]}" TARGET_DIRECTORY="${TARGET_DIRECTORIES[i]}" COMPRESSED_TARGET="${TARGET_NAME}-lib.tar.gz" + echo "${TARGET_NAME} release found: ${TARGET_BUILT[i]} (Target direcory: $TARGET_DIRECTORY)" + if [ "${TARGET_BUILT[i]}" == "true" ]; then echo "Downloading cached target: $TARGET_NAME, sha: $TARGET_SHA" gh release download ${GITHUB_RELEASE_NAME} --pattern "${COMPRESSED_TARGET}" --clobber @@ -819,10 +807,17 @@ jobs: then echo "Extracting cached target: ${COMPRESSED_TARGET}" tar -xf ${COMPRESSED_TARGET} - FOLDER_HASH=`find ./${TARGET_DIRECTORY} -xdev -type f -print0 | LC_COLLATE=C sort -z | xargs -0 cat | md5sum | awk '{print $1}'` + FOLDER_HASH=$(find ./${TARGET_DIRECTORY} -xdev -type f -exec cat {} + | md5sum | awk '{print $1}') + echo "Cached target folder hash: ${FOLDER_HASH}" - jsonStrHash=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"binFolderHash\" : \"${FOLDER_HASH}\" }" <<< "{}"` - jsonStr+=`echo ${jsonStrHash}` + jsonStrHash=$(jq -n \ + --arg name "$TARGET_NAME" \ + --arg sha "$TARGET_SHA" \ + --arg binFolderName "$TARGET_DIRECTORY" \ + --arg binFolderHash "$FOLDER_HASH" \ + '{name: $name, sha: $sha, binFolderName: $binFolderName, binFolderHash: $binFolderHash}') + + jsonStr=$(jq -c --argjson obj "$jsonStrHash" '. + [$obj]' <<< "$jsonStr") echo "Target extracted to `pwd`/${TARGET_DIRECTORY}" echo "Contents of target folder `pwd`/${TARGET_DIRECTORY}" @@ -848,7 +843,10 @@ jobs: jq <<< $jsonStr > hash.json # check if release.json file already available - releaseJsonStr="" + releaseJsonStr="[]" + + set +e + jsonFound=`gh release view ${GITHUB_RELEASE_NAME} --json assets --jq "any(.assets[]; .name == \"release.json\")"` if [ $? -eq 0 ] then @@ -861,6 +859,8 @@ jobs: fi fi + set -e + for i in "${!listOfTargetNamesToBuild[@]}"; do TARGET_NAME="${listOfTargetNamesToBuild[i]}" TARGET_SHA="${listOfTargetSHAsToBuild[i]}" @@ -870,7 +870,6 @@ jobs: echo "Compiling $TARGET_OS version $TARGET_SHA of $TARGET_NAME $BUILD_TYPE mode" cmake --build . --target $TARGET_NAME --config $BUILD_TYPE -j - cmake --install $TARGET_NAME echo "Starting to upload target: ${COMPRESSED_TARGET}" # Upload released target @@ -884,23 +883,34 @@ jobs: # Get target url TARGET_BINARY=`jq ".assets[] | select(.name == \"${COMPRESSED_TARGET}\") | .url" <<< ${RELEASE_ASSETS}` - releaseFound=`jq "(. | select(.name == \"${TARGET_NAME}\")" <<< ${releaseJsonStr}` + releaseFound=$(jq ".[] | select(.name == \"${TARGET_NAME}\")" <<<${releaseJsonStr}) if [ ${#releaseFound} -gt 0 ] then echo "Updating release.json file with target ${TARGET_NAME}" - releaseJsonStr=`jq "(. | select(.name == \"${TARGET_NAME}\")).sha |= \"${TARGET_SHA}\" " <<< ${releaseJsonStr}` - releaseJsonStr=`jq "(. | select(.name == \"${TARGET_NAME}\")).binFolderName |= \"${TARGET_DIRECTORY}\" " <<< ${releaseJsonStr}` - releaseJsonStr=`jq "(. | select(.name == \"${TARGET_NAME}\")).source |= \"${TARGET_SOURCE}\" " <<< ${releaseJsonStr}` - releaseJsonStr=`jq "(. | select(.name == \"${TARGET_NAME}\")).binary |= \"${TARGET_BINARY}\" " <<< ${releaseJsonStr}` + releaseJsonStr=$(jq --arg name "${TARGET_NAME}" \ + --arg sha "${TARGET_SHA}" \ + --arg binFolderName "${TARGET_DIRECTORY}" \ + --arg source "${TARGET_SOURCE}" \ + --arg binary "${TARGET_BINARY}" \ + '(.[] | select(.name == $name)) |= (.sha = $sha | .binFolderName = $binFolderName | .source = $source | .binary = $binary)' <<<"${releaseJsonStr}") + else echo "Adding target ${TARGET_NAME} to release.json file" - jsonStrTarget=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"source\" : ${TARGET_SOURCE}, \"binary\" : ${TARGET_BINARY} }" <<< "{}"` - releaseJsonStr+=`echo ${jsonStrTarget}` + jsonStrTarget=$(jq -n \ + --arg name "${TARGET_NAME}" \ + --arg sha "${TARGET_SHA}" \ + --arg binFolderName "${TARGET_DIRECTORY}" \ + --arg source "${TARGET_SOURCE}" \ + --arg binary "${TARGET_BINARY}" \ + '{name: $name, sha: $sha, binFolderName: $binFolderName, source: $source, binary: $binary}') + + releaseJsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$releaseJsonStr") fi # Upload JSON file release jq <<< $releaseJsonStr > release.json + echo "Uploading JSON file release" ${{ matrix.target-save-data.upload-release-json }} done @@ -947,9 +957,6 @@ jobs: IFS=$'\n' read -d '|' -ra TARGET_BUILT <<< "$(echo -ne '${{ steps.cache-library-directory.outputs.TARGET_BUILT }}|')" IFS=$'\n' read -d '|' -ra TARGET_DIRECTORIES <<< "$(echo -ne '${{ needs.create-targets-matrix.outputs.TARGET_DIRECTORIES }}|')" - # Setting this to not fail the workflow while checking release tag - set +e - GITHUB_RELEASE_NAME=${TARGET_OS}-${BRANCH}-${BUILD_TYPE} RELEASE_ASSETS=`gh release view ${GITHUB_RELEASE_NAME} --json assets` @@ -967,7 +974,7 @@ jobs: if [ "${TARGET_BUILT[i]}" == "true" ]; then # Compare uncompressed original hash of the binary folder with current to see if any changes has been made originalHash=`jq "select(.name == \"${TARGET_NAME}\") | .binFolderHash" hash.json` - currentHash=`find ./${TARGET_DIRECTORY} -xdev -type f -print0 | LC_COLLATE=C sort -z | xargs -0 cat | md5sum | awk '{print $1}'` + currentHash=$(find ./${TARGET_DIRECTORY} -xdev -type f -exec cat {} + | md5sum | awk '{print $1}') if [[ "${originalHash}" != \"${currentHash}\" ]]; then echo "originalHash: ${originalHash}" @@ -985,7 +992,7 @@ jobs: # Get target url TARGET_BINARY=`jq ".assets[] | select(.name == \"${COMPRESSED_TARGET}\") | .url" <<< ${RELEASE_ASSETS}` jsonStrTarget=`jq ". + { \"name\": \"${TARGET_NAME}\", \"sha\" : \"${TARGET_SHA}\", \"binFolderName\" : \"${TARGET_DIRECTORY}\", \"source\" : ${TARGET_SOURCE}, \"binary\" : ${TARGET_BINARY} }" <<< "{}"` - jsonStr+=`echo ${jsonStrTarget}` + jsonStr=$(jq -c --argjson obj "$jsonStrTarget" '. + [$obj]' <<<"$jsonStr") done # Print final JSON file diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 16628e8..6a1010e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -7,6 +7,7 @@ jobs: build: env: GRPC_BUILD_ENABLE_CCACHE: "ON" + CMAKE_NO_VERBOSE: "ON" runs-on: ${{matrix.host}} strategy: fail-fast: false @@ -28,7 +29,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - submodules: "recursive" + submodules: "true" + + - name: Checkout submodules + run: git submodule foreach git submodule update --init --recommend-shallow - name: Set clang as the default compiler if: ${{ matrix.host == 'ubuntu-latest' }} @@ -56,6 +60,7 @@ jobs: echo "CMAKE_GENERATOR=Ninja" >> $GITHUB_ENV - name: Add Darwin toolchain + if: ${{ matrix.target == 'OSX'}} run: rustup target add x86_64-apple-darwin - name: Add iOS toolchain diff --git a/build/Android/CMakeLists.txt b/build/Android/CMakeLists.txt index 3d03758..928d66e 100644 --- a/build/Android/CMakeLists.txt +++ b/build/Android/CMakeLists.txt @@ -20,6 +20,8 @@ if(NOT CMAKE_ANDROID_NDK) endif() endif() +set(ANDROID_USE_LEGACY_TOOLCHAIN_FILE OFF CACHE BOOL "Disabling legacy toolchain behavior") + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_ANDROID_NDK}/build/cmake/android.toolchain.cmake") project(ANDROID_SHARED_THIRD_PARTY LANGUAGES C CXX) @@ -34,11 +36,12 @@ endif() set(_CMAKE_COMMON_CACHE_ARGS ${_CMAKE_COMMON_CACHE_ARGS} -DCMAKE_ANDROID_NDK:PATH=${CMAKE_ANDROID_NDK} -DANDROID_ABI:STRING=${ANDROID_ABI} - -DANDROID_PLATFORM_LEVEL:STRING=${ANDROID_PLATFORM_LEVEL} -DANDROID_NATIVE_API_LEVEL:STRING=${ANDROID_NATIVE_API_LEVEL} + -DANDROID_PLATFORM_LEVEL:STRING=${ANDROID_PLATFORM_LEVEL} -DANDROID_PLATFORM:STRING=${ANDROID_PLATFORM} - -DANDROID_TOOLCHAIN:STRING=${ANDROID_TOOLCHAIN} -DANDROID_STL:STRING=c++_static + -DANDROID_TOOLCHAIN:STRING=${ANDROID_TOOLCHAIN} + -DANDROID_USE_LEGACY_TOOLCHAIN_FILE:BOOL=${ANDROID_USE_LEGACY_TOOLCHAIN_FILE} -DOPENSSL_USE_STATIC_LIBS:BOOL=TRUE ) diff --git a/build/CommonTargets.CMake b/build/CommonTargets.CMake index 038fb63..e68d0e0 100644 --- a/build/CommonTargets.CMake +++ b/build/CommonTargets.CMake @@ -15,7 +15,7 @@ set(COMMON_GRPC_CACHE_ARGS -DBUILD_TESTING:BOOL=OFF -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} -DgRPC_BUILD_CSHARP_EXT:BOOL=OFF - -DgRPC_BUILD_GRPC_CPP_PLUGIN:BOOL=ON + -DgRPC_BUILD_GRPC_CPP_PLUGIN:BOOL=OFF -DgRPC_BUILD_GRPC_CSHARP_PLUGIN:BOOL=OFF -DgRPC_BUILD_GRPC_NODE_PLUGIN:BOOL=OFF -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN:BOOL=OFF @@ -44,6 +44,7 @@ if(CMAKE_CROSSCOMPILING) CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=Release ${COMMON_GRPC_CACHE_ARGS} ) @@ -54,7 +55,7 @@ if(CMAKE_CROSSCOMPILING) else() set(GRPC_HOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc") set(PROTOBUF_TARGET grpc) - set(gRPC_BUILD_CODEGEN ON) + set(gRPC_BUILD_CODEGEN OFF) endif() set(PROTOBUF_HOST_PREFIX "${GRPC_HOST_PREFIX}") @@ -577,6 +578,7 @@ ExternalProject_Add(protobuf-plugin CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= -Dabsl_DIR:PATH=${absl_HOST_DIR} + -DCMAKE_BUILD_TYPE:STRING=Release -DProtobuf_DIR:PATH=${Protobuf_HOST_DIR} -Dprotobuf_MODULE_COMPATIBLE:BOOL=ON -DTW_USE_EXTERNAL_PROTOC:BOOL=TRUE @@ -616,7 +618,6 @@ ExternalProject_Add(wallet-core ${_CMAKE_COMMON_CACHE_ARGS} ${_BOOST_CACHE_ARGS} ${_OPENSSL_CACHE_ARGS} - ${_GRPC_CACHE_ARGS} PATCH_COMMAND ${_WALLET_CORE_GENERATE_SCRIPT} --target-os=${_WALLET_CORE_ARCH} --arch-abi=${WALLET_CORE_ARCH_ABI} --protobuf-dir=${PROTOBUF_HOST_PREFIX} --out-dir= --plugin-dir=${ProtobufPlugin_DIR}/bin DEPENDS Boost grpc openssl protobuf-plugin ) @@ -633,7 +634,6 @@ ExternalProject_Add(libssh2 -DCRYPTO_BACKEND:STRING=OpenSSL -DZLIB_ROOT:PATH=${ZLIB_ROOT} -DZLIB_USE_STATIC_LIBS:BOOL=ON - -DCRYPTO_BACKEND:STRING=OpenSSL ${_OPENSSL_CACHE_ARGS} DEPENDS openssl zlib ) diff --git a/build/apple.toolchain.cmake b/build/apple.toolchain.cmake index 536ed35..5ed518f 100644 --- a/build/apple.toolchain.cmake +++ b/build/apple.toolchain.cmake @@ -876,7 +876,7 @@ else() endif() if(NOT ${CMAKE_SYSTEM_PROCESSOR} STREQUAL ${CMAKE_HOST_SYSTEM_PROCESSOR}) - set(CMAKE_SYSTEM_NAME ${CMAKE_HOST_SYSTEM_NAME}) + set(CMAKE_SYSTEM_NAME ${TOOLCHAIN_SYSTEM_NAME}) endif() # Note that only Xcode 7+ supports the newer more specific: diff --git a/ipfs-lite-cpp b/ipfs-lite-cpp index 0af410c..604402d 160000 --- a/ipfs-lite-cpp +++ b/ipfs-lite-cpp @@ -1 +1 @@ -Subproject commit 0af410ccd245ffd2dc4aaf383753faa2046f10cb +Subproject commit 604402dbb4a28346d03d73405384746029d8c16d