From 492b8f0a0a0f75d501a013df285aa962112c8e1a Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 15:33:35 +0300 Subject: [PATCH 01/23] ci: Try and use artifacts to cache prebuilt NSS --- .github/actions/nss/action.yml | 48 +++++++++++++++++++++++++++++----- .github/workflows/bench.yml | 1 + .github/workflows/check.yml | 1 + .github/workflows/mutants.yml | 1 + 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index b5989476f0..219f30b4b5 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -8,6 +8,9 @@ inputs: minimum-version: description: "Minimum required version of NSS" required: true + token: + description: 'A Github PAT' + required: true runs: using: composite @@ -96,16 +99,36 @@ runs: repository: nss-dev/nspr path: nspr + - name: Get head revisions + if: env.BUILD_NSS == '1' + shell: bash + run: | + NSS_HEAD=$(git -C nss rev-parse HEAD) + NSPR_HEAD=$(git -C nspr rev-parse HEAD) + echo "NSS_HEAD=$NSS_HEAD" >> "$GITHUB_ENV" + echo "NSPR_HEAD=$NSPR_HEAD" >> "$GITHUB_ENV" + + - name: Download prebuilt NSS + if: env.BUILD_NSS == '1' + id: prebuilt + uses: actions/download-artifact@v4 + with: + name: nss-{{ runner.os }}-{{ env.NSS_HEAD }}-{{ env.NSPR_HEAD }} + path: nss/dist + github-token: ${{ inputs.token }} + - name: Install build dependencies (Linux) + needs: prebuilt shell: bash - if: runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted' + if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted' env: DEBIAN_FRONTEND: noninteractive run: sudo apt-get install -y --no-install-recommends gyp ninja-build - name: Install build dependencies (MacOS) + needs: prebuilt shell: bash - if: runner.os == 'MacOS' && env.BUILD_NSS == '1' + if: needs.prebuilt.outputs.download-path == '' && runner.os == 'MacOS' && env.BUILD_NSS == '1' run: | brew install ninja echo "gyp-next>=0.18.1" > req.txt @@ -113,8 +136,9 @@ runs: echo "$(python3 -m site --user-base)/bin" >> "$GITHUB_PATH" - name: Install build dependencies (Windows) + needs: prebuilt shell: bash - if: runner.os == 'Windows' && env.BUILD_NSS == '1' + if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' run: | # shellcheck disable=SC2028 { @@ -126,14 +150,16 @@ runs: python3 -m pip install -r req.txt - name: Set up MSVC (Windows) - if: runner.os == 'Windows' && env.BUILD_NSS == '1' + needs: prebuilt + if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' uses: ilammy/msvc-dev-cmd@v1 # TODO: Would like to pin this, but the Mozilla org allowlist requires "ilammy/msvc-dev-cmd@v1*" # uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up build environment (Windows) + needs: prebuilt shell: bash - if: runner.os == 'Windows' && env.BUILD_NSS == '1' + if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' run: | { echo "GYP_MSVS_OVERRIDE_PATH=$VSINSTALLDIR" @@ -144,8 +170,9 @@ runs: rm /usr/bin/link.exe || true - name: Build + needs: prebuilt shell: bash - if: env.BUILD_NSS == '1' + if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' run: | if [ "${{ inputs.type }}" != "Debug" ]; then # We want to do an optimized build for accurate CPU profiling, but @@ -166,3 +193,12 @@ runs: env: NSS_DIR: ${{ github.workspace }}/nss NSPR_DIR: ${{ github.workspace }}/nspr + + - name: Cache NSS build + needs: prebuilt + if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' + uses: actions/upload-artifact@v4 + with: + name: nss-{{ runner.os }}-{{ env.NSS_HEAD }}-{{ env.NSPR_HEAD }} + path: nss/dist + github-token: ${{ inputs.token }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d758aff65b..883713a962 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -55,6 +55,7 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Build neqo run: | diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e2861e3400..6e1ffe6e51 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -61,6 +61,7 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Build run: | diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 202f53d652..0d52a3deba 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -30,6 +30,7 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Install Rust uses: ./.github/actions/rust From 3c994d60943ca7c5a0faa7be2c49abe32560cc43 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 15:36:42 +0300 Subject: [PATCH 02/23] needs --- .github/actions/nss/action.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 219f30b4b5..7f25a8d3f8 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -118,7 +118,6 @@ runs: github-token: ${{ inputs.token }} - name: Install build dependencies (Linux) - needs: prebuilt shell: bash if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted' env: @@ -126,7 +125,6 @@ runs: run: sudo apt-get install -y --no-install-recommends gyp ninja-build - name: Install build dependencies (MacOS) - needs: prebuilt shell: bash if: needs.prebuilt.outputs.download-path == '' && runner.os == 'MacOS' && env.BUILD_NSS == '1' run: | @@ -136,7 +134,6 @@ runs: echo "$(python3 -m site --user-base)/bin" >> "$GITHUB_PATH" - name: Install build dependencies (Windows) - needs: prebuilt shell: bash if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' run: | @@ -150,14 +147,12 @@ runs: python3 -m pip install -r req.txt - name: Set up MSVC (Windows) - needs: prebuilt if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' uses: ilammy/msvc-dev-cmd@v1 # TODO: Would like to pin this, but the Mozilla org allowlist requires "ilammy/msvc-dev-cmd@v1*" # uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up build environment (Windows) - needs: prebuilt shell: bash if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' run: | @@ -170,7 +165,6 @@ runs: rm /usr/bin/link.exe || true - name: Build - needs: prebuilt shell: bash if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' run: | @@ -195,7 +189,6 @@ runs: NSPR_DIR: ${{ github.workspace }}/nspr - name: Cache NSS build - needs: prebuilt if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' uses: actions/upload-artifact@v4 with: From 6c6d0f682d5e327ee0daaaa19ccd402d4bc4052c Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 16:00:43 +0300 Subject: [PATCH 03/23] actions/cache --- .github/actions/nss/action.yml | 41 ++++++++++++++++------------------ .github/workflows/bench.yml | 1 - .github/workflows/check.yml | 1 - .github/workflows/mutants.yml | 1 - 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 7f25a8d3f8..1e85c027d5 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -8,9 +8,6 @@ inputs: minimum-version: description: "Minimum required version of NSS" required: true - token: - description: 'A Github PAT' - required: true runs: using: composite @@ -108,25 +105,33 @@ runs: echo "NSS_HEAD=$NSS_HEAD" >> "$GITHUB_ENV" echo "NSPR_HEAD=$NSPR_HEAD" >> "$GITHUB_ENV" - - name: Download prebuilt NSS + - name: Cache NSS + id: cache if: env.BUILD_NSS == '1' - id: prebuilt - uses: actions/download-artifact@v4 + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - name: nss-{{ runner.os }}-{{ env.NSS_HEAD }}-{{ env.NSPR_HEAD }} path: nss/dist - github-token: ${{ inputs.token }} + key: nss-${{ runner.os }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }} + + -name: Check if build is needed + if: env.BUILD_NSS == '1' + shell: bash + run: | + if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then + echo "NSS build not needed" + echo "BUILD_NSS=0" >> "$GITHUB_ENV" + fi - name: Install build dependencies (Linux) shell: bash - if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted' + if: runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted' env: DEBIAN_FRONTEND: noninteractive run: sudo apt-get install -y --no-install-recommends gyp ninja-build - name: Install build dependencies (MacOS) shell: bash - if: needs.prebuilt.outputs.download-path == '' && runner.os == 'MacOS' && env.BUILD_NSS == '1' + if: runner.os == 'MacOS' && env.BUILD_NSS == '1' run: | brew install ninja echo "gyp-next>=0.18.1" > req.txt @@ -135,7 +140,7 @@ runs: - name: Install build dependencies (Windows) shell: bash - if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' + if: runner.os == 'Windows' && env.BUILD_NSS == '1' run: | # shellcheck disable=SC2028 { @@ -147,14 +152,14 @@ runs: python3 -m pip install -r req.txt - name: Set up MSVC (Windows) - if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' + if: runner.os == 'Windows' && env.BUILD_NSS == '1' uses: ilammy/msvc-dev-cmd@v1 # TODO: Would like to pin this, but the Mozilla org allowlist requires "ilammy/msvc-dev-cmd@v1*" # uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 - name: Set up build environment (Windows) shell: bash - if: needs.prebuilt.outputs.download-path == '' && runner.os == 'Windows' && env.BUILD_NSS == '1' + if: runner.os == 'Windows' && env.BUILD_NSS == '1' run: | { echo "GYP_MSVS_OVERRIDE_PATH=$VSINSTALLDIR" @@ -166,7 +171,7 @@ runs: - name: Build shell: bash - if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' + if: env.BUILD_NSS == '1' run: | if [ "${{ inputs.type }}" != "Debug" ]; then # We want to do an optimized build for accurate CPU profiling, but @@ -187,11 +192,3 @@ runs: env: NSS_DIR: ${{ github.workspace }}/nss NSPR_DIR: ${{ github.workspace }}/nspr - - - name: Cache NSS build - if: needs.prebuilt.outputs.download-path == '' && env.BUILD_NSS == '1' - uses: actions/upload-artifact@v4 - with: - name: nss-{{ runner.os }}-{{ env.NSS_HEAD }}-{{ env.NSPR_HEAD }} - path: nss/dist - github-token: ${{ inputs.token }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 883713a962..d758aff65b 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -55,7 +55,6 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - token: ${{ secrets.GITHUB_TOKEN }} - name: Build neqo run: | diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6e1ffe6e51..e2861e3400 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -61,7 +61,6 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - token: ${{ secrets.GITHUB_TOKEN }} - name: Build run: | diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index 0d52a3deba..202f53d652 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -30,7 +30,6 @@ jobs: uses: ./.github/actions/nss with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - token: ${{ secrets.GITHUB_TOKEN }} - name: Install Rust uses: ./.github/actions/rust From 44bf010ad25d5f72dc32ca9a3a826d25f7f978ef Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 16:04:01 +0300 Subject: [PATCH 04/23] Fix --- .github/actions/nss/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 1e85c027d5..62d21e554e 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -107,14 +107,14 @@ runs: - name: Cache NSS id: cache - if: env.BUILD_NSS == '1' + if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted' uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: nss/dist key: nss-${{ runner.os }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }} - -name: Check if build is needed - if: env.BUILD_NSS == '1' + - name: Check if build is needed + if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted' shell: bash run: | if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then From 15edba8a08a5ad5ac44986d0b2ea902855f6389f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 16:11:40 +0300 Subject: [PATCH 05/23] Fix --- .github/actions/nss/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 62d21e554e..d149d04de5 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -110,7 +110,7 @@ runs: if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted' uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: - path: nss/dist + path: dist key: nss-${{ runner.os }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }} - name: Check if build is needed From fa328994e596311dd042c3bea20be941a4616d88 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 16:42:12 +0300 Subject: [PATCH 06/23] no sccache --- .github/actions/nss/action.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index d149d04de5..9875763724 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -59,29 +59,6 @@ runs: echo "System NSS is suitable: $NSS_VERSION" echo "BUILD_NSS=0" >> "$GITHUB_ENV" - - name: Use sccache - # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the RUSTC_WRAPPER environment variable is set - # (which every "use" of this action needs to therefore set) - if: env.RUSTC_WRAPPER != 'sccache' - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - - - name: Enable sscache - shell: bash - run: | - if [ "${{ runner.os }}" != "Windows" ]; then - # TODO: Figure out how to make this work on Windows - echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" - echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV" - fi - echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - if [ "$GITHUB_WORKFLOW" ]; then - echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - fi - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - - name: Checkout NSS if: env.BUILD_NSS == '1' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 From 5e1e590ce9858950859a75386d6530c5b78a1887 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 16:53:58 +0300 Subject: [PATCH 07/23] sccache is killing our cache --- .github/actions/nss/action.yml | 23 +++++++++++++++++++++++ .github/actions/rust/action.yml | 32 ++++++++++++++++---------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 9875763724..95483d9f5e 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -59,6 +59,29 @@ runs: echo "System NSS is suitable: $NSS_VERSION" echo "BUILD_NSS=0" >> "$GITHUB_ENV" + # - name: Use sccache + # # Apparently the action can't be installed twice in the same workflow, so check if + # # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # # (which every "use" of this action needs to therefore set) + # if: env.RUSTC_WRAPPER != 'sccache' + # uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + + # - name: Enable sscache + # shell: bash + # run: | + # if [ "${{ runner.os }}" != "Windows" ]; then + # # TODO: Figure out how to make this work on Windows + # echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" + # echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV" + # fi + # echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + # echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + # if [ "$GITHUB_WORKFLOW" ]; then + # echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" + # fi + # echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + # echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + - name: Checkout NSS if: env.BUILD_NSS == '1' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index c96ec7b269..e2502dcf8b 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -33,23 +33,23 @@ runs: components: ${{ inputs.components }} targets: ${{ inputs.targets }} - - name: Use sccache - # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the RUSTC_WRAPPER environment variable is set - # (which every "use" of this action needs to therefore set) - if: env.RUSTC_WRAPPER != 'sccache' - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + # - name: Use sccache + # # Apparently the action can't be installed twice in the same workflow, so check if + # # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # # (which every "use" of this action needs to therefore set) + # if: env.RUSTC_WRAPPER != 'sccache' + # uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - - name: Enable sscache - shell: bash - run: | - echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - if [ "$GITHUB_WORKFLOW" ]; then - echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - fi - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + # - name: Enable sscache + # shell: bash + # run: | + # echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + # echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + # if [ "$GITHUB_WORKFLOW" ]; then + # echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" + # fi + # echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + # echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - name: Set up MSVC (Windows) if: runner.os == 'Windows' From cb82aeff864e939cf0f7e009adf005d03c8aedf6 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 17:09:51 +0300 Subject: [PATCH 08/23] Set env --- .github/actions/nss/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 95483d9f5e..3c2b1b958e 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -171,7 +171,6 @@ runs: - name: Build shell: bash - if: env.BUILD_NSS == '1' run: | if [ "${{ inputs.type }}" != "Debug" ]; then # We want to do an optimized build for accurate CPU profiling, but @@ -188,7 +187,7 @@ runs: echo "$NSS_OUT/lib" >> "$GITHUB_PATH" echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV" [ "$SCCACHE_CC" ] && [ "$SCCACHE_CXX" ] && export CC="$SCCACHE_CC" CXX="$SCCACHE_CXX" - $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static + [ "$BUILD_NSS" == "1" ] && $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static env: NSS_DIR: ${{ github.workspace }}/nss NSPR_DIR: ${{ github.workspace }}/nspr From 6a30d2f654f5b73111f5aa7a8fa6931b26b450c4 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 17:12:10 +0300 Subject: [PATCH 09/23] Env --- .github/actions/nss/action.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 3c2b1b958e..31e3b15d17 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -169,16 +169,9 @@ runs: # See https://github.com/ilammy/msvc-dev-cmd#name-conflicts-with-shell-bash rm /usr/bin/link.exe || true - - name: Build + - name: Set up environment shell: bash run: | - if [ "${{ inputs.type }}" != "Debug" ]; then - # We want to do an optimized build for accurate CPU profiling, but - # we also want debug symbols and frame pointers for that, which the normal optimized NSS - # build process doesn't provide. - OPT="-o" - [ "${{ runner.os }}" != "Windows" ] && export CFLAGS="-ggdb3 -fno-omit-frame-pointer" - fi NSS_TARGET="${{ inputs.type }}" echo "NSS_TARGET=$NSS_TARGET" >> "$GITHUB_ENV" NSS_OUT="$NSS_DIR/../dist/$NSS_TARGET" @@ -186,8 +179,20 @@ runs: echo "DYLD_FALLBACK_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV" echo "$NSS_OUT/lib" >> "$GITHUB_PATH" echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV" - [ "$SCCACHE_CC" ] && [ "$SCCACHE_CXX" ] && export CC="$SCCACHE_CC" CXX="$SCCACHE_CXX" - [ "$BUILD_NSS" == "1" ] && $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static env: NSS_DIR: ${{ github.workspace }}/nss NSPR_DIR: ${{ github.workspace }}/nspr + + - name: Build + shell: bash + if: env.BUILD_NSS == '1' + run: | + if [ "${{ inputs.type }}" != "Debug" ]; then + # We want to do an optimized build for accurate CPU profiling, but + # we also want debug symbols and frame pointers for that, which the normal optimized NSS + # build process doesn't provide. + OPT="-o" + [ "${{ runner.os }}" != "Windows" ] && export CFLAGS="-ggdb3 -fno-omit-frame-pointer" + fi + [ "$SCCACHE_CC" ] && [ "$SCCACHE_CXX" ] && export CC="$SCCACHE_CC" CXX="$SCCACHE_CXX" + $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static From f15c066fd96c9e1b12930f3a5e79c48dedd9e54f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 17 Sep 2024 17:31:50 +0300 Subject: [PATCH 10/23] NSS_PREBUILT --- .github/actions/nss/action.yml | 1 + neqo-crypto/build.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 31e3b15d17..d0d1b172bd 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -179,6 +179,7 @@ runs: echo "DYLD_FALLBACK_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV" echo "$NSS_OUT/lib" >> "$GITHUB_PATH" echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV" + echo "NSS_PREBUILT=1" >> "$GITHUB_ENV" env: NSS_DIR: ${{ github.workspace }}/nss NSPR_DIR: ${{ github.workspace }}/nspr diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs index 34cc842b5e..69ab267f6b 100644 --- a/neqo-crypto/build.rs +++ b/neqo-crypto/build.rs @@ -317,7 +317,10 @@ fn setup_standalone(nss: &str) -> Vec { "The NSS_DIR environment variable is expected to be an absolute path." ); - build_nss(nss.clone()); + // If NSS_PREBUILT is set, we assume that the NSS libraries are already built. + if env::var("NSS_PREBUILT").unwrap_or_default() != "1" { + build_nss(nss.clone()); + } // $NSS_DIR/../dist/ let nssdist = nss.parent().unwrap().join("dist"); From 4962c1eadc7b88c08d9395de6da0d6364d5fc8b7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 08:47:02 +0300 Subject: [PATCH 11/23] Check if set --- neqo-crypto/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs index 69ab267f6b..9df3f460a2 100644 --- a/neqo-crypto/build.rs +++ b/neqo-crypto/build.rs @@ -318,7 +318,7 @@ fn setup_standalone(nss: &str) -> Vec { ); // If NSS_PREBUILT is set, we assume that the NSS libraries are already built. - if env::var("NSS_PREBUILT").unwrap_or_default() != "1" { + if env::var("NSS_PREBUILT").is_ok() { build_nss(nss.clone()); } From 7aecf53fc8f15fd5a3765596861c3651c647cf12 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 09:03:21 +0300 Subject: [PATCH 12/23] Cache on self-hosted runner --- .github/actions/nss/action.yml | 48 ++++++++++++++++++--------------- .github/actions/rust/action.yml | 36 ++++++++++++++----------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index d0d1b172bd..c247b5f290 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -59,28 +59,32 @@ runs: echo "System NSS is suitable: $NSS_VERSION" echo "BUILD_NSS=0" >> "$GITHUB_ENV" - # - name: Use sccache - # # Apparently the action can't be installed twice in the same workflow, so check if - # # it's already installed by checking if the RUSTC_WRAPPER environment variable is set - # # (which every "use" of this action needs to therefore set) - # if: env.RUSTC_WRAPPER != 'sccache' - # uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - - # - name: Enable sscache - # shell: bash - # run: | - # if [ "${{ runner.os }}" != "Windows" ]; then - # # TODO: Figure out how to make this work on Windows - # echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" - # echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV" - # fi - # echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - # echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - # if [ "$GITHUB_WORKFLOW" ]; then - # echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - # fi - # echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - # echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + - name: Use sccache + # Apparently the action can't be installed twice in the same workflow, so check if + # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # (which every "use" of this action needs to therefore set) + # + # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit + # is too small for this to be effective there. + if: env.RUSTC_WRAPPER != 'sccache' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' + uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + + - name: Enable sscache + if: env.RUSTC_WRAPPER != 'sccache' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' + shell: bash + run: | + if [ "${{ runner.os }}" != "Windows" ]; then + # TODO: Figure out how to make this work on Windows + echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" + echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV" + fi + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + if [ "$GITHUB_WORKFLOW" ]; then + echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" + fi + echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - name: Checkout NSS if: env.BUILD_NSS == '1' diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index e2502dcf8b..b2b22c580c 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -33,23 +33,27 @@ runs: components: ${{ inputs.components }} targets: ${{ inputs.targets }} - # - name: Use sccache - # # Apparently the action can't be installed twice in the same workflow, so check if - # # it's already installed by checking if the RUSTC_WRAPPER environment variable is set - # # (which every "use" of this action needs to therefore set) - # if: env.RUSTC_WRAPPER != 'sccache' - # uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + - name: Use sccache + # Apparently the action can't be installed twice in the same workflow, so check if + # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # (which every "use" of this action needs to therefore set) + # + # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit + # is too small for this to be effective there. + if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' + uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - # - name: Enable sscache - # shell: bash - # run: | - # echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - # echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - # if [ "$GITHUB_WORKFLOW" ]; then - # echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - # fi - # echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - # echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" + - name: Enable sscache + if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' + shell: bash + run: | + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" + if [ "$GITHUB_WORKFLOW" ]; then + echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" + fi + echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - name: Set up MSVC (Windows) if: runner.os == 'Windows' From d791b8bc23f3a2adf2869c70cf95ac7d2b3f6fce Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 09:26:53 +0300 Subject: [PATCH 13/23] Fixes --- .github/actions/nss/action.yml | 4 +--- .github/actions/rust/action.yml | 2 -- Cargo.toml | 5 +++++ neqo-crypto/build.rs | 16 ++++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index c247b5f290..d53be71ac0 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -83,8 +83,6 @@ runs: if [ "$GITHUB_WORKFLOW" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - name: Checkout NSS if: env.BUILD_NSS == '1' @@ -115,7 +113,7 @@ runs: uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 with: path: dist - key: nss-${{ runner.os }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }} + key: nss-${{ runner.os }}-${{ inputs.type }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }} - name: Check if build is needed if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted' diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index b2b22c580c..6b71d52ca1 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -47,8 +47,6 @@ runs: if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' shell: bash run: | - echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" if [ "$GITHUB_WORKFLOW" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi diff --git a/Cargo.toml b/Cargo.toml index 815470bddd..8c7f0e1f6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,11 @@ nursery = { level = "warn", priority = -1 } pedantic = { level = "warn", priority = -1 } multiple_crate_versions = "allow" +# Optimize build dependencies, because bindgen and proc macros / style +# compilation take more to run than to build otherwise. +[profile.dev.build-override] +opt-level = 1 + [profile.release] lto = "fat" diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs index 9df3f460a2..7e34f57320 100644 --- a/neqo-crypto/build.rs +++ b/neqo-crypto/build.rs @@ -109,14 +109,14 @@ fn get_bash() -> PathBuf { ) } -fn build_nss(dir: PathBuf) { +fn build_nss(dir: PathBuf, nsstarget: &str) { let mut build_nss = vec![ String::from("./build.sh"), String::from("-Ddisable_tests=1"), // Generate static libraries in addition to shared libraries. String::from("--static"), ]; - if !is_debug() { + if nsstarget == "Release" { build_nss.push(String::from("-o")); } if let Ok(d) = env::var("NSS_JOBS") { @@ -317,18 +317,18 @@ fn setup_standalone(nss: &str) -> Vec { "The NSS_DIR environment variable is expected to be an absolute path." ); - // If NSS_PREBUILT is set, we assume that the NSS libraries are already built. - if env::var("NSS_PREBUILT").is_ok() { - build_nss(nss.clone()); - } - // $NSS_DIR/../dist/ let nssdist = nss.parent().unwrap().join("dist"); println!("cargo:rerun-if-env-changed=NSS_TARGET"); let nsstarget = env::var("NSS_TARGET") .unwrap_or_else(|_| fs::read_to_string(nssdist.join("latest")).unwrap()); - let nsstarget = nssdist.join(nsstarget.trim()); + // If NSS_PREBUILT is set, we assume that the NSS libraries are already built. + if env::var("NSS_PREBUILT").is_ok() { + build_nss(nss, &nsstarget); + } + + let nsstarget = nssdist.join(nsstarget.trim()); let includes = get_includes(&nsstarget, &nssdist); let nsslibdir = nsstarget.join("lib"); From 391556b960e5a59e2de6b92f5b17ce851adc6221 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 09:59:44 +0300 Subject: [PATCH 14/23] Fixes --- .github/actions/nss/action.yml | 2 +- .github/actions/rust/action.yml | 2 +- .github/workflows/check.yml | 10 +++++++--- .github/workflows/clippy.yml | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index d53be71ac0..e3696b61bb 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -80,7 +80,7 @@ runs: fi echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV" - if [ "$GITHUB_WORKFLOW" ]; then + if [ "${{ runner.environment }}" == "github-hosted" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index 6b71d52ca1..cff0e31599 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -47,7 +47,7 @@ runs: if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' shell: bash run: | - if [ "$GITHUB_WORKFLOW" ]; then + if [ "${{ runner.environment }}" == "github-hosted" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 49dfb8ec80..de746dface 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -47,8 +47,8 @@ jobs: - uses: ./.github/actions/rust with: version: ${{ matrix.rust-toolchain }} - components: clippy, llvm-tools-preview - tools: cargo-llvm-cov, cargo-nextest, cargo-hack, cargo-fuzz + components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' }} + tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, '}} cargo-nextest, cargo-fuzz token: ${{ secrets.GITHUB_TOKEN }} - id: nss-version @@ -70,7 +70,11 @@ jobs: - name: Run tests and determine coverage run: | # shellcheck disable=SC2086 - RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info + if [ "${{ matrix.rust-toolchain }}" == "stable" ]; then + RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info + else + RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast + fi cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run - name: Run client/server transfer diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index a1ef1ed6ba..c323f79048 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -27,7 +27,7 @@ jobs: - uses: ./.github/actions/rust with: components: clippy - tools: cargo-hack, cargo-fuzz + tools: cargo-hack token: ${{ secrets.GITHUB_TOKEN }} - id: nss-version From 530b4802b2246fe0f53c5fa33f33fc6d728652bd Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 10:08:41 +0300 Subject: [PATCH 15/23] Fixes --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index de746dface..2685740cd7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -47,8 +47,8 @@ jobs: - uses: ./.github/actions/rust with: version: ${{ matrix.rust-toolchain }} - components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' }} - tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, '}} cargo-nextest, cargo-fuzz + components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' || '' }} + tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, ' || '' }} cargo-nextest, cargo-fuzz token: ${{ secrets.GITHUB_TOKEN }} - id: nss-version From 36741c8105a534d63016ebc93719cbf23f0f22e1 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 10:19:56 +0300 Subject: [PATCH 16/23] Run fuzz in parallel --- .github/workflows/check.yml | 6 +----- .github/workflows/fuzz.yml | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/fuzz.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2685740cd7..42e3946dfa 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -48,7 +48,7 @@ jobs: with: version: ${{ matrix.rust-toolchain }} components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' || '' }} - tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, ' || '' }} cargo-nextest, cargo-fuzz + tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, ' || '' }} cargo-nextest token: ${{ secrets.GITHUB_TOKEN }} - id: nss-version @@ -62,10 +62,6 @@ jobs: run: | # shellcheck disable=SC2086 cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci - # Check that the fuzz targets also build - if [ ${{ startsWith(matrix.rust-toolchain, 'nightly') && 'nightly' }} == 'nightly' ]; then - cargo +${{ matrix.rust-toolchain }} fuzz check - fi - name: Run tests and determine coverage run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000000..2af42bf28a --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,38 @@ +name: Fuzz +on: + workflow_dispatch: + pull_request: + branches: ["main"] + paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"] + merge_group: +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: ./.github/actions/rust + with: + version: nightly + tools: cargo-fuzz + token: ${{ secrets.GITHUB_TOKEN }} + + - id: nss-version + run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT" + + - uses: ./.github/actions/nss + with: + minimum-version: ${{ steps.nss-version.outputs.minimum }} + + # Check that the fuzz targets build + - run: cargo fuzz check From d1469fd1904b349724c5b5f77a8f6b9a0fc0065d Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 10:21:53 +0300 Subject: [PATCH 17/23] Invert --- neqo-crypto/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neqo-crypto/build.rs b/neqo-crypto/build.rs index 7e34f57320..b61a8e92af 100644 --- a/neqo-crypto/build.rs +++ b/neqo-crypto/build.rs @@ -324,7 +324,7 @@ fn setup_standalone(nss: &str) -> Vec { .unwrap_or_else(|_| fs::read_to_string(nssdist.join("latest")).unwrap()); // If NSS_PREBUILT is set, we assume that the NSS libraries are already built. - if env::var("NSS_PREBUILT").is_ok() { + if env::var("NSS_PREBUILT").is_err() { build_nss(nss, &nsstarget); } From e28210a6bc3e4e02290bf6101d7d3f823c74103d Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 10:32:23 +0300 Subject: [PATCH 18/23] fuzz-bench --- .github/actions/nss/action.yml | 4 +++- .github/workflows/check.yml | 1 - .github/workflows/{fuzz.yml => fuzz-bench.yml} | 7 ++++--- 3 files changed, 7 insertions(+), 5 deletions(-) rename .github/workflows/{fuzz.yml => fuzz-bench.yml} (86%) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index e3696b61bb..ae57a74810 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -120,8 +120,10 @@ runs: shell: bash run: | if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then - echo "NSS build not needed" + echo "Using cached prebuilt NSS" echo "BUILD_NSS=0" >> "$GITHUB_ENV" + else + echo "Building NSS from source" fi - name: Install build dependencies (Linux) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 42e3946dfa..f4a9c7df00 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -71,7 +71,6 @@ jobs: else RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast fi - cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run - name: Run client/server transfer run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz-bench.yml similarity index 86% rename from .github/workflows/fuzz.yml rename to .github/workflows/fuzz-bench.yml index 2af42bf28a..6ffb3d1cbb 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz-bench.yml @@ -1,4 +1,4 @@ -name: Fuzz +name: Fuzz & Bench on: workflow_dispatch: pull_request: @@ -17,7 +17,7 @@ permissions: contents: read jobs: - fuzz: + fuzz-bench: runs-on: ubuntu-latest steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 @@ -34,5 +34,6 @@ jobs: with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - # Check that the fuzz targets build + # Check that the fuzz and bench targets build - run: cargo fuzz check + - run: cargo bench --features bench --no-run From 349dc2745c8df625060c1567eb1a7515b8b9f7c5 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 10:44:12 +0300 Subject: [PATCH 19/23] SCCACHE_INSTALLED && build -> check --- .github/actions/nss/action.yml | 7 ++++--- .github/actions/rust/action.yml | 7 ++++--- .github/workflows/check.yml | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index ae57a74810..1e3b4ee79f 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -61,18 +61,19 @@ runs: - name: Use sccache # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # it's already installed by checking if the SCCACHE_INSTALLED environment variable is set # (which every "use" of this action needs to therefore set) # # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit # is too small for this to be effective there. - if: env.RUSTC_WRAPPER != 'sccache' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' + if: env.SCCACHE_INSTALLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache - if: env.RUSTC_WRAPPER != 'sccache' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' + if: env.SCCACHE_INSTALLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' shell: bash run: | + echo "SCCACHE_INSTALLED=1" >> "$GITHUB_ENV" if [ "${{ runner.os }}" != "Windows" ]; then # TODO: Figure out how to make this work on Windows echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index cff0e31599..63e0cfce64 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -35,18 +35,19 @@ runs: - name: Use sccache # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the RUSTC_WRAPPER environment variable is set + # it's already installed by checking if the SCCACHE_INSTALLED environment variable is set # (which every "use" of this action needs to therefore set) # # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit # is too small for this to be effective there. - if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' + if: env.SCCACHE_INSTALLED != '1' && runner.environment != 'github-hosted' uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache - if: env.RUSTC_WRAPPER != 'sccache' && runner.environment != 'github-hosted' + if: env.SCCACHE_INSTALLED != 'sccache' && runner.environment != 'github-hosted' shell: bash run: | + echo "SCCACHE_INSTALLED=1" >> "$GITHUB_ENV" if [ "${{ runner.environment }}" == "github-hosted" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f4a9c7df00..fb7877b9ee 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -58,10 +58,10 @@ jobs: with: minimum-version: ${{ steps.nss-version.outputs.minimum }} - - name: Build + - name: Check run: | # shellcheck disable=SC2086 - cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci + cargo +${{ matrix.rust-toolchain }} check $BUILD_TYPE --all-targets --features ci - name: Run tests and determine coverage run: | From fc1b2bfa4f12747a9c66e5830075266ba85a5c27 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 11:02:04 +0300 Subject: [PATCH 20/23] Fixes --- .github/actions/nss/action.yml | 7 +++---- .github/actions/rust/action.yml | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 1e3b4ee79f..6f09de0368 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -61,19 +61,18 @@ runs: - name: Use sccache # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the SCCACHE_INSTALLED environment variable is set + # it's already installed by checking if the SCCACHE_ENABLED environment variable is set # (which every "use" of this action needs to therefore set) # # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit # is too small for this to be effective there. - if: env.SCCACHE_INSTALLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' + if: env.SCCACHE_ENABLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache - if: env.SCCACHE_INSTALLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted' shell: bash run: | - echo "SCCACHE_INSTALLED=1" >> "$GITHUB_ENV" + echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV" if [ "${{ runner.os }}" != "Windows" ]; then # TODO: Figure out how to make this work on Windows echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV" diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index 63e0cfce64..ce0118cb1e 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -35,19 +35,18 @@ runs: - name: Use sccache # Apparently the action can't be installed twice in the same workflow, so check if - # it's already installed by checking if the SCCACHE_INSTALLED environment variable is set + # it's already installed by checking if the SCCACHE_ENABLED environment variable is set # (which every "use" of this action needs to therefore set) # # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit # is too small for this to be effective there. - if: env.SCCACHE_INSTALLED != '1' && runner.environment != 'github-hosted' + if: env.SCCACHE_ENABLED != '1' && runner.environment != 'github-hosted' uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache - if: env.SCCACHE_INSTALLED != 'sccache' && runner.environment != 'github-hosted' shell: bash run: | - echo "SCCACHE_INSTALLED=1" >> "$GITHUB_ENV" + echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV" if [ "${{ runner.environment }}" == "github-hosted" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi From b15f464cdc3627bd06fde789aecf159781736578 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 11:05:10 +0300 Subject: [PATCH 21/23] Fixes --- .github/actions/nss/action.yml | 1 + .github/actions/rust/action.yml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/nss/action.yml b/.github/actions/nss/action.yml index 6f09de0368..b8f7470f38 100644 --- a/.github/actions/nss/action.yml +++ b/.github/actions/nss/action.yml @@ -70,6 +70,7 @@ runs: uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache + if: env.BUILD_NSS == '1' && runner.environment != 'github-hosted' shell: bash run: | echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV" diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index ce0118cb1e..64c155e14d 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -44,14 +44,15 @@ runs: uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - name: Enable sscache + if: runner.environment != 'github-hosted' shell: bash run: | echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV" + echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" if [ "${{ runner.environment }}" == "github-hosted" ]; then echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" fi - echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" - echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV" - name: Set up MSVC (Windows) if: runner.os == 'Windows' From 9370a329dd0a13a0fd41b4ec01b260c9d95881f7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 12:00:51 +0300 Subject: [PATCH 22/23] Don't update rustup --- .github/actions/rust/action.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index 64c155e14d..0f47e8fb2b 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -21,11 +21,6 @@ inputs: runs: using: composite steps: - - name: Upgrade rustup (MacOS) - shell: bash - if: runner.os == 'MacOS' - run: brew update && brew upgrade rustup - - name: Install Rust uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 # master with: From 595cf04ac1f66476bb9bb9426836dad567c7ed4b Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 18 Sep 2024 12:26:36 +0300 Subject: [PATCH 23/23] Compile less --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d758aff65b..8b959fb2f2 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -58,8 +58,8 @@ jobs: - name: Build neqo run: | - cargo "+$TOOLCHAIN" bench --features bench --no-run - cargo "+$TOOLCHAIN" build --release + cargo "+$TOOLCHAIN" bench --workspace --features bench --no-run + cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server - name: Build msquic run: |