Add workflow doing comparative testing of toolchain branches across m…

…ultiple targets
Windows-on-ARM-Experiments · Jan 18, 2025 · a627078 · a627078
1 parent af94e50
commit a627078
Show file tree

Hide file tree

Showing 18 changed files with 427 additions and 126 deletions.
diff --git a/.github/scripts/binutils/build.sh b/.github/scripts/binutils/build.sh
@@ -9,7 +9,7 @@ cd $BINUTILS_BUILD_PATH
 
 if [[ "$RUN_CONFIG" = 1 ]] || [[ ! -f "$BINUTILS_BUILD_PATH/Makefile" ]]; then
     echo "::group::Configure binutils"
-        rm -rf $BINUTILS_BUILD_PATH/*
+        $ROOT_PATH/.github/scripts/binutils/delete-build.sh
 
         if [[ "$DEBUG" = 1 ]]; then
             HOST_OPTIONS="$HOST_OPTIONS \
@@ -57,7 +57,7 @@ if [[ "$RUN_INSTALL" = 1 ]]; then
     echo "::group::Install binutils"
         make install
         if [[ "$DELETE_BUILD" = 1 ]]; then
-            rm -rf $BINUTILS_BUILD_PATH
+            $ROOT_PATH/.github/scripts/binutils/delete-build.sh
         fi
     echo "::endgroup::"
 fi

diff --git a/.github/scripts/binutils/delete-build.sh b/.github/scripts/binutils/delete-build.sh
@@ -1,5 +1,11 @@
 #!/bin/bash
 
-source `dirname ${BASH_SOURCE[0]}`/config.sh
+source `dirname ${BASH_SOURCE[0]}`/../config.sh
 
-rm -rf $BINUTILS_BUILD_PATH
+BINUTILS_BUILD_PATH=$BUILD_PATH/binutils
+
+echo "::group::Delete binutils build"
+    rm -rf $BINUTILS_BUILD_PATH/*
+echo "::endgroup::"
+
+echo 'Success!'
diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -71,13 +71,20 @@ if [[ "$RUN_BOOTSTRAP" = 1 || "$RESET_SOURCES" = 1 ]]; then
     $ROOT_PATH/.github/scripts/install-libraries.sh
 fi
 
-$ROOT_PATH/.github/scripts/toolchain/build-gcc.sh
-
-if [[ "$PLATFORM" =~ (mingw|cygwin) ]]; then
-    $ROOT_PATH/.github/scripts/toolchain/build-mingw.sh
-fi
-if [[ "$PLATFORM" =~ cygwin ]]; then
-    $ROOT_PATH/.github/scripts/toolchain/build-cygwin.sh 2
+if [[ "$TEST" = 1 ]]; then
+    DELETE_BUILD=0 \
+    $ROOT_PATH/.github/scripts/toolchain/build-gcc.sh
+else
+    $ROOT_PATH/.github/scripts/toolchain/build-gcc.sh
+fi
+
+if [[ "$TEST" = 0 ]]; then
+    if [[ "$PLATFORM" =~ (mingw|cygwin) ]]; then
+        $ROOT_PATH/.github/scripts/toolchain/build-mingw.sh
+    fi
+    if [[ "$PLATFORM" =~ cygwin ]]; then
+        $ROOT_PATH/.github/scripts/toolchain/build-cygwin.sh 2
+    fi
 fi
 
 if [[ "$CCACHE" = 1 ]]; then

diff --git a/.github/scripts/config.sh b/.github/scripts/config.sh
@@ -40,8 +40,8 @@ else
 fi
 
 PROCESSOR=$(uname --processor)
-BUILD=${BUILD:-$PROCESSOR-pc-linux-gnu}
-HOST=${HOST:-$PROCESSOR-pc-linux-gnu}
+BUILD=${BUILD:-$PROCESSOR-linux-gnu}
+HOST=${HOST:-$PROCESSOR-linux-gnu}
 TARGET=$ARCH-$PLATFORM
 TOOLCHAIN_NAME=${TOOLCHAIN_NAME:-$ARCH-$PLATFORM-$CRT}
 
@@ -105,6 +105,7 @@ else
 fi
 
 DEBUG=${DEBUG:-0} # Enable debug build.
+TEST=${TEST:-0} # Enable build for testing.
 CCACHE=${CCACHE:-0} # Enable usage of ccache.
 RUN_BOOTSTRAP=${RUN_BOOTSTRAP:-0} # Bootstrap dependencies during the build.
 UPDATE_SOURCES=${UPDATE_SOURCES:-0} # Update source code repositories.

diff --git a/.github/scripts/get-cache-keys.sh b/.github/scripts/get-cache-keys.sh
@@ -8,8 +8,10 @@ GCC_SHA=$($ROOT_PATH/.github/scripts/get-repository-sha.sh $GCC_REPO $GCC_BRANCH
 MINGW_SHA=$($ROOT_PATH/.github/scripts/get-repository-sha.sh $MINGW_REPO $MINGW_BRANCH)
 BINUTILS_SCRIPTS_SHA=$($ROOT_PATH/.github/scripts/get-files-sha.sh $ROOT_PATH $ROOT_PATH/.github/scripts/binutils)
 TOOLCHAIN_SCRIPTS_SHA=$($ROOT_PATH/.github/scripts/get-files-sha.sh $ROOT_PATH $ROOT_PATH/.github/scripts/toolchain)
+TIMESTAMP=$(date -u --iso-8601=seconds)
 
+echo "timestamp=$TIMESTAMP" >> "$GITHUB_OUTPUT"
 echo "toolchain-cache-key=$TOOLCHAIN_NAME-toolchain-$WORKFLOW_SHA-$BINUTILS_SHA-$GCC_SHA-$MINGW_SHA-$BINUTILS_SCRIPTS_SHA-$TOOLCHAIN_SCRIPTS_SHA" >> "$GITHUB_OUTPUT"
 echo "runtime-cache-key=$TOOLCHAIN_NAME-runtime-$WORKFLOW_SHA-$BINUTILS_SHA-$GCC_SHA-$MINGW_SHA-$BINUTILS_SCRIPTS_SHA-$TOOLCHAIN_SCRIPTS_SHA" >> "$GITHUB_OUTPUT"
-echo "ccache-key=$TARGET-ccache-$(date -u --iso-8601=seconds)" >> "$GITHUB_OUTPUT"
+echo "ccache-key=$TARGET-ccache-$TIMESTAMP" >> "$GITHUB_OUTPUT"
 echo "ccache-restore-keys=$TARGET-ccache-" >> "$GITHUB_OUTPUT"
diff --git a/.github/scripts/get-paths.sh b/.github/scripts/get-paths.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+source `dirname ${BASH_SOURCE[0]}`/config.sh
+
+echo "root-path-env=$ROOT_PATH" >> $GITHUB_OUTPUT
+echo "build-path-env=$BUILD_PATH" >> $GITHUB_OUTPUT
+echo "ccache-dir-path-env=$CCACHE_DIR_PATH" >> $GITHUB_OUTPUT
+echo "artifact-path-env=$ARTIFACT_PATH" >> $GITHUB_OUTPUT
+
+if [ -x "$(command -v wslpath)" ]; then
+    echo "root-path-host=`wslpath -w $ROOT_PATH`" >> $GITHUB_OUTPUT
+    echo "build-path-host=`wslpath -w $BUILD_PATH`" >> $GITHUB_OUTPUT
+    echo "ccache-dir-path-host=`wslpath -w $CCACHE_DIR_PATH`" >> $GITHUB_OUTPUT
+    echo "artifact-path-host=`wslpath -w $ARTIFACT_PATH`" >> $GITHUB_OUTPUT
+else
+    echo "root-path-host=$ROOT_PATH" >> $GITHUB_OUTPUT
+    echo "build-path-host=$BUILD_PATH" >> $GITHUB_OUTPUT
+    echo "ccache-dir-path-host=$CCACHE_DIR_PATH" >> $GITHUB_OUTPUT
+    echo "artifact-path-host=$ARTIFACT_PATH" >> $GITHUB_OUTPUT
+fi
diff --git a/.github/scripts/toolchain/build-gcc.sh b/.github/scripts/toolchain/build-gcc.sh
@@ -9,7 +9,7 @@ cd $GCC_BUILD_PATH
 
 if [[ "$RUN_CONFIG" = 1 ]] || [[ ! -f "$GCC_BUILD_PATH/Makefile" ]]; then
     echo "::group::Configure GCC"
-        rm -rf $GCC_BUILD_PATH/*
+        $ROOT_PATH/.github/scripts/toolchain/delete-gcc-build.sh
 
         if [[ "$DEBUG" = 1 ]]; then
             HOST_OPTIONS="$HOST_OPTIONS \
@@ -117,7 +117,7 @@ if [[ "$RUN_INSTALL" = 1 ]]; then
     echo "::group::Install GCC"
         make install
         if [[ "$DELETE_BUILD" = 1 ]]; then
-            rm -rf $GCC_BUILD_PATH
+            $ROOT_PATH/.github/scripts/toolchain/delete-gcc-build.sh
         fi
     echo "::endgroup::"
 fi

diff --git a/.github/scripts/toolchain/compare-gcc-results.sh b/.github/scripts/toolchain/compare-gcc-results.sh
@@ -2,20 +2,31 @@
 
 source `dirname ${BASH_SOURCE[0]}`/../config.sh
 
-BASELINE_TAG=$1
-TEST_TAG=$2
-BASELINE_PATH=$ARTIFACT_PATH/gcc-tests-$BASELINE_TAG
-TEST_PATH=$ARTIFACT_PATH/gcc-tests-$TEST_TAG
-RESULTS_PATH=$ARTIFACT_PATH/gcc-tests-results
+BASELINE_PATH=$1
+CHANGES_PATH=$2
+RESULTS_PATH=$3
 
 mkdir -p $RESULTS_PATH
 
 echo "::group::Compare GCC tests results"
-    for TEST_FILE in $TEST_PATH/*.sum; do
-        BASELINE_FIILE=$BASELINE_PATH/`basename $TEST_FILE`
-        $SOURCE_PATH/gcc/contrib/compare_tests $BASELINE_FIILE $TEST_FILE > \
-          $RESULTS_PATH/`basename $TEST_FILE .sum`.diff && true
+    for CHANGES_FILE in $CHANGES_PATH/*.sum; do
+        FILE=`basename $CHANGES_FILE`
+        BASELINE_FILE=$BASELINE_PATH/$FILE
+        RESULT_FILE=${FILE%.sum}.diff
+        $SOURCE_PATH/gcc/contrib/compare_tests $BASELINE_FILE $CHANGES_FILE > \
+          $RESULTS_PATH/$RESULT_FILE && true
     done
 echo "::endgroup::"
 
+echo "::group::Create GCC tests results summary"
+    rm -rf $RESULTS_PATH/summary.md
+    SUMMARY_FILE=$RESULTS_PATH/summary.md
+    for RESULT_FILE in $RESULTS_PATH/*.diff; do
+        echo "### Test results for \`$(basename $RESULT_FILE)\`" >> $SUMMARY_FILE
+        echo "" >> $SUMMARY_FILE
+        echo "\`\`\`" >> $SUMMARY_FILE
+        cat $RESULT_FILE >> $SUMMARY_FILE
+        echo "\`\`\`" >> $SUMMARY_FILE
+        echo "" >> $SUMMARY_FILE
+    done
 echo 'Success!'
diff --git a/.github/scripts/toolchain/create-gcc-summary.sh b/.github/scripts/toolchain/create-gcc-summary.sh
@@ -24,6 +24,7 @@ print_summary() {
     fi
 
     echo "### $TITLE"
+    echo ""
     echo "| Metric                | Count                 |"
     echo "|-----------------------|-----------------------|"
     echo "| Expected passes       | $EXPECTED_PASSES      |"

diff --git a/.github/scripts/toolchain/delete-gcc-build.sh b/.github/scripts/toolchain/delete-gcc-build.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+source `dirname ${BASH_SOURCE[0]}`/../config.sh
+
+GCC_BUILD_PATH=$BUILD_PATH/gcc
+
+echo "::group::Delete GCC build"
+    rm -rf $GCC_BUILD_PATH/*
+echo "::endgroup::"
+
+echo 'Success!'
diff --git a/.github/scripts/toolchain/execute-gcc-tests.sh b/.github/scripts/toolchain/execute-gcc-tests.sh
@@ -11,7 +11,6 @@ TARGET_BOARD=${4:-wsl-sim}
 HOST_BOARD=${5:-}
 
 GCC_BUILD_PATH=$BUILD_PATH/gcc
-TEST_RESULTS_PATH=$ARTIFACT_PATH/$DIR
 
 PATH="$TOOLCHAIN_PATH/bin:$PATH"
 
@@ -38,12 +37,17 @@ echo "::group::Execute GCC tests"
         CHECK_TEST_FRAMEWORK=1 \
         || echo "Error"
 
-    mkdir -p $TEST_RESULTS_PATH
-    rm -rf $TEST_RESULTS_PATH/*
+    mkdir -p $DIR
+    rm -rf $DIR/*
 
     for FILE in `find $GCC_BUILD_PATH -path '*testsuite*.log' -or -path '*testsuite*.sum'`; do
-        cp $FILE $TEST_RESULTS_PATH/
+        cp $FILE $DIR/
     done
 echo "::endgroup::"
 
+echo "::group::Create GCC tests summary"
+     $ROOT_PATH/.github/scripts/toolchain/create-gcc-summary.sh $DIR >> $DIR/summary.md
+     $ROOT_PATH/.github/scripts/toolchain/group-gcc-test-failures.sh $DIR >> $DIR/most-frequent-groups.md
+echo "::endgroup::"
+
 echo 'Success!'
diff --git a/.github/scripts/toolchain/group-gcc-test-failures.sh b/.github/scripts/toolchain/group-gcc-test-failures.sh
@@ -5,8 +5,7 @@ source `dirname ${BASH_SOURCE[0]}`/../config.sh
 DIR=$1
 
 python3 $ROOT_PATH/.github/scripts/toolchain/group-gcc-test-failures.py \
-    --dir $ARTIFACT_PATH/$DIR >> \
-    $ARTIFACT_PATH/$DIR/grouped-test-failures.md
+    --dir $DIR >> $DIR/grouped-test-failures.md
 
-cat $ARTIFACT_PATH/$DIR/grouped-test-failures.md |
+cat $DIR/grouped-test-failures.md |
     $ROOT_PATH/.github/scripts/toolchain/extract-most-frequent-failures.sh 10
diff --git a/.github/scripts/toolchain/pack-gcc-build.sh b/.github/scripts/toolchain/pack-gcc-build.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+source `dirname ${BASH_SOURCE[0]}`/../config.sh
+
+GCC_BUILD_PATH=$BUILD_PATH/gcc
+
+echo "::group::Pack GCC build"
+    mkdir -p $ARTIFACT_PATH
+    tar czf $ARTIFACT_PATH/gcc-build.tar.gz -C $GCC_BUILD_PATH .
+echo "::endgroup::"
+
+echo 'Success!'
diff --git a/.github/workflows/advanced.yml b/.github/workflows/advanced.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
   workflow_dispatch:
     inputs:
       binutils_branch:
@@ -133,14 +132,14 @@ jobs:
       fail-fast: false
       matrix:
         arch: [aarch64, x86_64]
-        platform: [w64-mingw32, pc-linux-gnu, pc-cygwin]
+        platform: [w64-mingw32, linux-gnu, pc-cygwin]
         crt: [msvcrt, ucrt, libc]
         exclude:
           - platform: w64-mingw32
             crt: libc
-          - platform: pc-linux-gnu
+          - platform: linux-gnu
             crt: msvcrt
-          - platform: pc-linux-gnu
+          - platform: linux-gnu
             crt: ucrt
           - platform: pc-cygwin
             crt: ucrt
@@ -171,7 +170,7 @@ jobs:
           fetch-depth: 0
 
       - name: Get cache keys
-        id: cache-keys
+        id: get-cache-keys
         run: |
           .github/scripts/get-cache-keys.sh
 
@@ -181,7 +180,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.ARTIFACT_PATH }}/${{ env.TOOLCHAIN_PACKAGE_NAME }}
-          key: ${{ steps.cache-keys.outputs.toolchain-cache-key }}
+          key: ${{ steps.get-cache-keys.outputs.toolchain-cache-key }}
           enableCrossOsArchive: true
 
       - name: Cache runtime
@@ -190,7 +189,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.RELATIVE_ARTIFACT_PATH }}/${{ env.RUNTIME_PACKAGE_NAME }}
-          key: ${{ steps.cache-keys.outputs.runtime-cache-key }}
+          key: ${{ steps.get-cache-keys.outputs.runtime-cache-key }}
           enableCrossOsArchive: true
 
       - name: Checkout binutils
@@ -210,7 +209,7 @@ jobs:
           path: ${{ env.SOURCE_PATH }}/gcc
 
       - name: Checkout MinGW
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'linux-gnu' }}
         uses: actions/checkout@v4
         with:
           repository: ${{ env.MINGW_REPO }}
@@ -265,8 +264,8 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ${{ env.CCACHE_DIR_PATH }}
-          key: ${{ steps.cache-keys.outputs.ccache-key }}
-          restore-keys: ${{ steps.cache-keys.outputs.ccache-restore-keys }}
+          key: ${{ steps.get-cache-keys.outputs.ccache-key }}
+          restore-keys: ${{ steps.get-cache-keys.outputs.ccache-restore-keys }}
 
       - name: Build binutils
         if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' }}
@@ -287,12 +286,12 @@ jobs:
           .github/scripts/binutils/delete-build.sh
 
       - name: Install cross headers and libraries
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform == 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform == 'linux-gnu' }}
         run: |
           .github/scripts/toolchain/install-cross-headers-libs.sh
 
       - name: Build MinGW headers
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'linux-gnu' }}
         run: |
           .github/scripts/toolchain/build-mingw-headers.sh
 
@@ -302,12 +301,12 @@ jobs:
           .github/scripts/toolchain/install-cygwin-headers.sh
 
       - name: Build GCC stage1
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'linux-gnu' }}
         run: |
           .github/scripts/toolchain/build-gcc-stage1.sh
 
       - name: Build MinGW CRT
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'linux-gnu' }}
         run: |
           .github/scripts/toolchain/build-mingw-crt.sh
 
@@ -342,7 +341,7 @@ jobs:
           .github/scripts/toolchain/build-gcc.sh
 
       - name: Build MinGW
-        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'pc-linux-gnu' }}
+        if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && matrix.platform != 'linux-gnu' }}
         run: |
           .github/scripts/toolchain/build-mingw.sh
 
@@ -356,7 +355,7 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: ${{ env.CCACHE_DIR_PATH }}
-          key: ccache-${{ steps.cache-keys.outputs.ccache-key }}
+          key: ccache-${{ steps.get-cache-keys.outputs.ccache-key }}
 
       - name: Pack toolchain
         if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' && env.PACK_TOOLCHAIN == 'true' }}
@@ -397,8 +396,8 @@ jobs:
       toolchain-artifact-name: ${{ env.PACK_TOOLCHAIN == 'true' && env.TOOLCHAIN_ARTIFACT_NAME || '' }}
       runtime-package-name: ${{ env.PACK_TOOLCHAIN == 'true' && env.RUNTIME_PACKAGE_NAME || '' }}
       runtime-artifact-name: ${{ env.PACK_TOOLCHAIN == 'true' && env.RUNTIME_ARTIFACT_NAME || '' }}
-      toolchain-cache-key: ${{ env.PACK_TOOLCHAIN == 'true' && steps.cache-keys.outputs.toolchain-cache-key || '' }}
-      runtime-cache-key: ${{ env.PACK_TOOLCHAIN == 'true' && steps.cache-keys.outputs.runtime-cache-key || '' }}
+      toolchain-cache-key: ${{ env.PACK_TOOLCHAIN == 'true' && steps.get-cache-keys.outputs.toolchain-cache-key || '' }}
+      runtime-cache-key: ${{ env.PACK_TOOLCHAIN == 'true' && steps.get-cache-keys.outputs.runtime-cache-key || '' }}
 
   build-aarch64-tests:
     needs: [build-toolchain]