From f8d8cc90fe1468e895673c996e81fe86661dde3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Wed, 17 Jul 2024 14:46:07 +0200 Subject: [PATCH] chore(ci): adapt benchmarks workflows to use slab-github-runner --- .github/workflows/boolean_benchmark.yml | 115 ++++++----- .github/workflows/core_crypto_benchmark.yml | 111 +++++----- .github/workflows/integer_benchmark.yml | 130 ------------ ...enchmark.yml => integer_cpu_benchmark.yml} | 121 +++++++---- .../workflows/integer_multi_bit_benchmark.yml | 130 ------------ .github/workflows/shortint_benchmark.yml | 128 ------------ ...nchmark.yml => shortint_cpu_benchmark.yml} | 133 +++++++----- .../workflows/signed_integer_benchmark.yml | 130 ------------ .../signed_integer_cpu_benchmark.yml | 191 ++++++++++++++++++ .../signed_integer_full_benchmark.yml | 136 ------------- .../signed_integer_multi_bit_benchmark.yml | 130 ------------ .github/workflows/start_benchmarks.yml | 123 ----------- .github/workflows/start_full_benchmarks.yml | 66 ------ 13 files changed, 487 insertions(+), 1157 deletions(-) delete mode 100644 .github/workflows/integer_benchmark.yml rename .github/workflows/{integer_full_benchmark.yml => integer_cpu_benchmark.yml} (57%) delete mode 100644 .github/workflows/integer_multi_bit_benchmark.yml delete mode 100644 .github/workflows/shortint_benchmark.yml rename .github/workflows/{shortint_full_benchmark.yml => shortint_cpu_benchmark.yml} (53%) delete mode 100644 .github/workflows/signed_integer_benchmark.yml create mode 100644 .github/workflows/signed_integer_cpu_benchmark.yml delete mode 100644 .github/workflows/signed_integer_full_benchmark.yml delete mode 100644 .github/workflows/signed_integer_multi_bit_benchmark.yml delete mode 100644 .github/workflows/start_benchmarks.yml delete mode 100644 .github/workflows/start_full_benchmarks.yml diff --git a/.github/workflows/boolean_benchmark.yml b/.github/workflows/boolean_benchmark.yml index 326efa7830..c8a9118bfc 100644 --- a/.github/workflows/boolean_benchmark.yml +++ b/.github/workflows/boolean_benchmark.yml @@ -3,30 +3,9 @@ name: Boolean benchmarks on: workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - # This input is not used in this workflow but still mandatory since a calling workflow could - # use it. If a triggering command include a user_inputs field, then the triggered workflow - # must include this very input, otherwise the workflow won't be called. - # See start_full_benchmarks.yml as example. - user_inputs: - description: "Type of benchmarks to run" - type: string - default: "weekly_benchmarks" + schedule: + # Weekly benchmarks will be triggered each Saturday at 1a.m. + - cron: '0 1 * * 6' env: CARGO_TERM_COLOR: always @@ -34,29 +13,53 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} jobs: - run-boolean-benchmarks: - name: Execute boolean benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + name: Setup instance (boolean-benchmarks) + runs-on: ubuntu-latest + if: github.event_name != 'schedule' || + (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + boolean-benchmarks: + name: Execute boolean benchmarks in EC2 + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + continue-on-error: true + steps: - name: Checkout tfhe-rs repo with tags uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: fetch-depth: 0 + - name: Get benchmark details + run: | + { + echo "BENCH_DATE=$(date --iso-8601=seconds)"; + echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; + echo "COMMIT_HASH=$(git describe --tags --dirty)"; + } >> "${GITHUB_ENV}" + - name: Set up home # "Install rust" step require root user to have a HOME directory which is not set. run: | @@ -73,14 +76,12 @@ jobs: - name: Parse results run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ + --hardware "hpc7a.96xlarge" \ + --project-version "${{ env.COMMIT_HASH }}" \ --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ + --commit-date "${{ env.COMMIT_DATE }}" \ --bench-date "${{ env.BENCH_DATE }}" \ --walk-subdirs \ --name-suffix avx512 \ @@ -129,8 +130,28 @@ jobs: uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 env: SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_MESSAGE: "Boolean benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + teardown-instance: + name: Teardown instance (boolean-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, boolean-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (boolean-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/core_crypto_benchmark.yml b/.github/workflows/core_crypto_benchmark.yml index 34d287c69d..edb8e0d4f2 100644 --- a/.github/workflows/core_crypto_benchmark.yml +++ b/.github/workflows/core_crypto_benchmark.yml @@ -3,30 +3,6 @@ name: Core crypto benchmarks on: workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - # This input is not used in this workflow but still mandatory since a calling workflow could - # use it. If a triggering command include a user_inputs field, then the triggered workflow - # must include this very input, otherwise the workflow won't be called. - # See start_full_benchmarks.yml as example. - user_inputs: - description: "Type of benchmarks to run" - type: string - default: "weekly_benchmarks" env: CARGO_TERM_COLOR: always @@ -34,29 +10,52 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} jobs: - run-core-crypto-benchmarks: - name: Execute core crypto benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + setup-instance: + name: Setup instance (core-crypto-benchmarks) + runs-on: ubuntu-latest + if: github.event_name != 'schedule' || + (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + core-crypto-benchmarks: + name: Execute core crypto benchmarks in EC2 + needs: setup-instance + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + steps: - name: Checkout tfhe-rs repo with tags uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: fetch-depth: 0 + - name: Get benchmark details + run: | + { + echo "BENCH_DATE=$(date --iso-8601=seconds)"; + echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; + echo "COMMIT_HASH=$(git describe --tags --dirty)"; + } >> "${GITHUB_ENV}" + - name: Set up home # "Install rust" step require root user to have a HOME directory which is not set. run: | @@ -75,14 +74,12 @@ jobs: - name: Parse results run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ + --hardware "hpc7a.96xlarge" \ + --project-version "${{ env.COMMIT_HASH }}" \ --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ + --commit-date "${{ env.COMMIT_DATE }}" \ --bench-date "${{ env.BENCH_DATE }}" \ --name-suffix avx512 \ --walk-subdirs \ @@ -121,8 +118,28 @@ jobs: uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 env: SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_MESSAGE: "PBS benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + teardown-instance: + name: Teardown instance (core-crypto-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, core-crypto-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (core-crypto-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/integer_benchmark.yml b/.github/workflows/integer_benchmark.yml deleted file mode 100644 index e1c1550b1b..0000000000 --- a/.github/workflows/integer_benchmark.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Run integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Integer benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - run-integer-benchmarks: - name: Execute integer benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Run benchmarks with AVX512 - run: | - make FAST_BENCH=TRUE bench_integer - - - name: Parse benchmarks to csv - run: | - make PARSE_INTEGER_BENCH_CSV_FILE=${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} \ - parse_integer_benches - - - name: Upload csv results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_csv_integer - path: ${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} - - - name: Parse results - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_integer - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Integer benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/integer_full_benchmark.yml b/.github/workflows/integer_cpu_benchmark.yml similarity index 57% rename from .github/workflows/integer_full_benchmark.yml rename to .github/workflows/integer_cpu_benchmark.yml index ff397bc4c3..6f224cd2b0 100644 --- a/.github/workflows/integer_full_benchmark.yml +++ b/.github/workflows/integer_cpu_benchmark.yml @@ -1,28 +1,20 @@ # Run all integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Integer full benchmarks +name: Integer benchmarks on: workflow_dispatch: inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - user_inputs: - description: "Type of benchmarks to run" - type: string - default: "weekly_benchmarks" + all_precisions: + description: "Run all precisions" + type: boolean + default: false + + schedule: + # Weekly benchmarks will be triggered each Saturday at 1a.m. + - cron: '0 1 * * 6' + # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. + # These benchmarks are far longer to execute hence the reason to run them only four time a year. + - cron: '0 4 25 MAR,JUN,SEP,DEC *' env: CARGO_TERM_COLOR: always @@ -30,21 +22,29 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + FAST_BENCH: TRUE jobs: prepare-matrix: name: Prepare operations matrix runs-on: ubuntu-latest + if: github.event_name != 'schedule' || + (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') outputs: op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} steps: - name: Weekly benchmarks - if: ${{ github.event.inputs.user_inputs == 'weekly_benchmarks' }} + if: github.event_name == 'workflow_dispatch' || + github.event.schedule == '0 1 * * 6' run: | echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" - name: Quarterly benchmarks - if: ${{ github.event.inputs.user_inputs == 'quarterly_benchmarks' }} + if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' run: | echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\", \"misc\"]" >> "${GITHUB_ENV}" @@ -53,11 +53,31 @@ jobs: run: | echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" + setup-instance: + name: Setup instance (integer-benchmarks) + needs: prepare-matrix + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + integer-benchmarks: name: Execute integer benchmarks for all operations flavor - needs: prepare-matrix - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + needs: [ prepare-matrix, setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} continue-on-error: true timeout-minutes: 1440 # 24 hours strategy: @@ -66,13 +86,6 @@ jobs: command: [ integer, integer_multi_bit] op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - name: Checkout tfhe-rs repo with tags uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: @@ -103,6 +116,11 @@ jobs: path: slab token: ${{ secrets.FHE_ACTIONS_TOKEN }} + - name: Should run benchmarks with all precisions + if: inputs.all_precisions + run: | + echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" + - name: Run benchmarks with AVX512 run: | make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_${{ matrix.command }} @@ -111,7 +129,7 @@ jobs: run: | python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ + --hardware "hpc7a.96xlarge" \ --project-version "${{ env.COMMIT_HASH }}" \ --branch ${{ github.ref_name }} \ --commit-date "${{ env.COMMIT_DATE }}" \ @@ -140,19 +158,34 @@ jobs: -d @${{ env.RESULTS_FILENAME }} \ ${{ secrets.SLAB_URL }} - slack-notification: - name: Slack Notification - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ failure() }} - needs: integer-benchmarks - steps: - - name: Notify + - name: Slack Notification + if: ${{ failure() }} continue-on-error: true uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 env: SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_MESSAGE: "Integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + teardown-instance: + name: Teardown instance (integer-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, integer-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/integer_multi_bit_benchmark.yml b/.github/workflows/integer_multi_bit_benchmark.yml deleted file mode 100644 index 5cd80f4b38..0000000000 --- a/.github/workflows/integer_multi_bit_benchmark.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Run integer benchmarks with multi-bit cryptographic parameters on an AWS instance and return parsed results to Slab CI bot. -name: Integer Multi-bit benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - run-integer-benchmarks: - name: Execute integer multi-bit benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Run multi-bit benchmarks with AVX512 - run: | - make FAST_BENCH=TRUE bench_integer_multi_bit - - - name: Parse benchmarks to csv - run: | - make PARSE_INTEGER_BENCH_CSV_FILE=${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} \ - parse_integer_benches - - - name: Upload csv results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_csv_integer - path: ${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} - - - name: Parse results - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_integer - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Integer benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/shortint_benchmark.yml b/.github/workflows/shortint_benchmark.yml deleted file mode 100644 index 168c7b6389..0000000000 --- a/.github/workflows/shortint_benchmark.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Run shortint benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Shortint benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - run-shortint-benchmarks: - name: Execute shortint benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Run benchmarks with AVX512 - run: | - make bench_shortint - - - name: Parse results - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Measure key sizes - run: | - make measure_shortint_key_sizes - - - name: Parse key sizes results - run: | - python3 ./ci/benchmark_parser.py tfhe/shortint_key_sizes.csv ${{ env.RESULTS_FILENAME }} \ - --key-sizes \ - --append-results - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_shortint - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Shortint benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/shortint_full_benchmark.yml b/.github/workflows/shortint_cpu_benchmark.yml similarity index 53% rename from .github/workflows/shortint_full_benchmark.yml rename to .github/workflows/shortint_cpu_benchmark.yml index 8aebde0ebd..8aa374b8ff 100644 --- a/.github/workflows/shortint_full_benchmark.yml +++ b/.github/workflows/shortint_cpu_benchmark.yml @@ -3,30 +3,13 @@ name: Shortint full benchmarks on: workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - # This input is not used in this workflow but still mandatory since a calling workflow could - # use it. If a triggering command include a user_inputs field, then the triggered workflow - # must include this very input, otherwise the workflow won't be called. - # See start_full_benchmarks.yml as example. - user_inputs: - description: "Type of benchmarks to run" - type: string - default: "weekly_benchmarks" + schedule: + # Weekly benchmarks will be triggered each Saturday at 1a.m. + - cron: '0 1 * * 6' + # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. + # These benchmarks are far longer to execute hence the reason to run them only four time a year. + - cron: '0 4 25 MAR,JUN,SEP,DEC *' + env: CARGO_TERM_COLOR: always @@ -34,24 +17,67 @@ env: ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUST_BACKTRACE: "full" RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} jobs: + prepare-matrix: + name: Prepare operations matrix + runs-on: ubuntu-latest + if: github.event_name != 'schedule' || + (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') + outputs: + op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} + steps: + - name: Weekly benchmarks + if: github.event_name == 'workflow_dispatch' || + github.event.schedule == '0 1 * * 6' + run: | + echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" + + - name: Quarterly benchmarks + if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' + run: | + echo "OP_FLAVOR=[\"default\", \"smart\", \"unchecked\"]" >> "${GITHUB_ENV}" + + - name: Set operation flavor output + id: set_op_flavor + run: | + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" + + setup-instance: + name: Setup instance (shortint-benchmarks) + needs: prepare-matrix + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + shortint-benchmarks: name: Execute shortint benchmarks for all operations flavor - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} + needs: [ prepare-matrix, setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + continue-on-error: true strategy: max-parallel: 1 matrix: - op_flavor: [ default, smart, unchecked ] + op_flavor: ${{ fromJson(needs.prepare-matrix.outputs.op_flavor) }} steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - name: Checkout tfhe-rs repo with tags uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 with: @@ -92,7 +118,7 @@ jobs: COMMIT_HASH="$(git describe --tags --dirty)" python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ + --hardware "hpc7a.96xlarge" \ --project-version "${COMMIT_HASH}" \ --branch ${{ github.ref_name }} \ --commit-date "${COMMIT_DATE}" \ @@ -134,19 +160,34 @@ jobs: -d @${{ env.RESULTS_FILENAME }} \ ${{ secrets.SLAB_URL }} - slack-notification: - name: Slack Notification - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ failure() }} - needs: shortint-benchmarks - steps: - - name: Notify + - name: Slack Notification + if: ${{ failure() }} continue-on-error: true uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 env: SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_MESSAGE: "Shortint full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + teardown-instance: + name: Teardown instance (shortint-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, shortint-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (shortint-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/signed_integer_benchmark.yml b/.github/workflows/signed_integer_benchmark.yml deleted file mode 100644 index 85713b207a..0000000000 --- a/.github/workflows/signed_integer_benchmark.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Run signed integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Signed Integer benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - run-integer-benchmarks: - name: Execute signed integer benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Run benchmarks with AVX512 - run: | - make FAST_BENCH=TRUE bench_signed_integer - - - name: Parse benchmarks to csv - run: | - make PARSE_INTEGER_BENCH_CSV_FILE=${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} \ - parse_integer_benches - - - name: Upload csv results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_csv_integer - path: ${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} - - - name: Parse results - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_integer - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Signed integer benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/signed_integer_cpu_benchmark.yml b/.github/workflows/signed_integer_cpu_benchmark.yml new file mode 100644 index 0000000000..56111dae4e --- /dev/null +++ b/.github/workflows/signed_integer_cpu_benchmark.yml @@ -0,0 +1,191 @@ +# Run all signed integer benchmarks on an AWS instance and return parsed results to Slab CI bot. +name: Signed Integer full benchmarks + +on: + workflow_dispatch: + inputs: + all_precisions: + description: "Run all precisions" + type: boolean + default: false + + schedule: + # Weekly benchmarks will be triggered each Saturday at 1a.m. + - cron: '0 1 * * 6' + # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. + # These benchmarks are far longer to execute hence the reason to run them only four time a year. + - cron: '0 4 25 MAR,JUN,SEP,DEC *' + +env: + CARGO_TERM_COLOR: always + RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUST_BACKTRACE: "full" + RUST_MIN_STACK: "8388608" + SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} + SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png + SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + FAST_BENCH: TRUE + +jobs: + prepare-matrix: + name: Prepare operations matrix + runs-on: ubuntu-latest + if: github.event_name != 'schedule' || + (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') + outputs: + op_flavor: ${{ steps.set_op_flavor.outputs.op_flavor }} + steps: + - name: Weekly benchmarks + if: github.event_name == 'workflow_dispatch' || + github.event.schedule == '0 1 * * 6' + run: | + echo "OP_FLAVOR=[\"default\"]" >> "${GITHUB_ENV}" + + - name: Quarterly benchmarks + if: github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' + run: | + echo "OP_FLAVOR=[\"default\", \"unchecked\"]" >> "${GITHUB_ENV}" + + - name: Set operation flavor output + id: set_op_flavor + run: | + echo "op_flavor=${{ toJSON(env.OP_FLAVOR) }}" >> "${GITHUB_OUTPUT}" + + setup-instance: + name: Setup instance (signed-integer-benchmarks) + needs: prepare-matrix + runs-on: ubuntu-latest + outputs: + runner-name: ${{ steps.start-instance.outputs.label }} + steps: + - name: Start instance + id: start-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: start + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + backend: aws + profile: bench + + signed-integer-benchmarks: + name: Execute signed integer benchmarks for all operations flavor + needs: [ prepare-matrix, setup-instance ] + runs-on: ${{ needs.setup-instance.outputs.runner-name }} + concurrency: + group: ${{ github.workflow }}_${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + continue-on-error: true + timeout-minutes: 1440 # 24 hours + strategy: + max-parallel: 1 + matrix: + command: [ integer, integer_multi_bit ] + op_flavor: [ default, unchecked ] + steps: + - name: Checkout tfhe-rs repo with tags + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + fetch-depth: 0 + + - name: Get benchmark details + run: | + { + echo "BENCH_DATE=$(date --iso-8601=seconds)"; + echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; + echo "COMMIT_HASH=$(git describe --tags --dirty)"; + } >> "${GITHUB_ENV}" + + - name: Set up home + # "Install rust" step require root user to have a HOME directory which is not set. + run: | + echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + + - name: Install rust + uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 + with: + toolchain: nightly + + - name: Checkout Slab repo + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + repository: zama-ai/slab + path: slab + token: ${{ secrets.FHE_ACTIONS_TOKEN }} + + - name: Should run benchmarks with all precisions + if: inputs.all_precisions + run: | + echo "FAST_BENCH=FALSE" >> "${GITHUB_ENV}" + + - name: Run benchmarks with AVX512 + run: | + make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_signed_${{ matrix.command }} + + - name: Parse results + run: | + python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ + --database tfhe_rs \ + --hardware "hpc7a.96xlarge" \ + --project-version "${{ env.COMMIT_HASH }}" \ + --branch ${{ github.ref_name }} \ + --commit-date "${{ env.COMMIT_DATE }}" \ + --bench-date "${{ env.BENCH_DATE }}" \ + --walk-subdirs \ + --name-suffix avx512 \ + --throughput + + - name: Upload parsed results artifact + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b + with: + name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }} + path: ${{ env.RESULTS_FILENAME }} + + - name: Send data to Slab + shell: bash + run: | + echo "Computing HMac on results file" + SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" + echo "Sending results to Slab..." + curl -v -k \ + -H "Content-Type: application/json" \ + -H "X-Slab-Repository: ${{ github.repository }}" \ + -H "X-Slab-Command: store_data_v2" \ + -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ + -d @${{ env.RESULTS_FILENAME }} \ + ${{ secrets.SLAB_URL }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Signed integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + teardown-instance: + name: Teardown instance (integer-benchmarks) + if: ${{ always() && needs.setup-instance.result != 'skipped' }} + needs: [ setup-instance, signed-integer-benchmarks ] + runs-on: ubuntu-latest + steps: + - name: Stop instance + id: stop-instance + uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261 + with: + mode: stop + github-token: ${{ secrets.SLAB_ACTION_TOKEN }} + slab-url: ${{ secrets.SLAB_BASE_URL }} + job-secret: ${{ secrets.JOB_SECRET }} + label: ${{ needs.setup-instance.outputs.runner-name }} + + - name: Slack Notification + if: ${{ failure() }} + continue-on-error: true + uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Instance teardown (signed-integer-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" diff --git a/.github/workflows/signed_integer_full_benchmark.yml b/.github/workflows/signed_integer_full_benchmark.yml deleted file mode 100644 index 920fb8013d..0000000000 --- a/.github/workflows/signed_integer_full_benchmark.yml +++ /dev/null @@ -1,136 +0,0 @@ -# Run all signed integer benchmarks on an AWS instance and return parsed results to Slab CI bot. -name: Signed Integer full benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - user_inputs: - description: "Type of benchmarks to run" - type: string - default: "weekly_benchmarks" - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - integer-benchmarks: - name: Execute signed integer benchmarks for all operations flavor - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - continue-on-error: true - timeout-minutes: 1440 # 24 hours - strategy: - max-parallel: 1 - matrix: - command: [ integer, integer_multi_bit ] - op_flavor: [ default, unchecked ] - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Get benchmark details - run: | - { - echo "BENCH_DATE=$(date --iso-8601=seconds)"; - echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"; - echo "COMMIT_HASH=$(git describe --tags --dirty)"; - } >> "${GITHUB_ENV}" - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Run benchmarks with AVX512 - run: | - make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_signed_${{ matrix.command }} - - - name: Parse results - run: | - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${{ env.COMMIT_HASH }}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${{ env.COMMIT_DATE }}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_${{ matrix.command }}_${{ matrix.op_flavor }} - path: ${{ env.RESULTS_FILENAME }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - slack-notification: - name: Slack Notification - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ failure() }} - needs: integer-benchmarks - steps: - - name: Notify - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Signed integer full benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/signed_integer_multi_bit_benchmark.yml b/.github/workflows/signed_integer_multi_bit_benchmark.yml deleted file mode 100644 index 94e6dbbfb3..0000000000 --- a/.github/workflows/signed_integer_multi_bit_benchmark.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Run signed integer benchmarks with multi-bit cryptographic parameters on an AWS instance and return parsed results to Slab CI bot. -name: Signed Integer Multi-bit benchmarks - -on: - workflow_dispatch: - inputs: - instance_id: - description: "Instance ID" - type: string - instance_image_id: - description: "Instance AMI ID" - type: string - instance_type: - description: "Instance product type" - type: string - runner_name: - description: "Action runner name" - type: string - request_id: - description: "Slab request ID" - type: string - -env: - CARGO_TERM_COLOR: always - RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json - PARSE_INTEGER_BENCH_CSV_FILE: tfhe_rs_integer_benches_${{ github.sha }}.csv - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUST_BACKTRACE: "full" - RUST_MIN_STACK: "8388608" - -jobs: - run-integer-benchmarks: - name: Execute signed integer multi-bit benchmarks in EC2 - runs-on: ${{ github.event.inputs.runner_name }} - if: ${{ !cancelled() }} - steps: - - name: Instance configuration used - run: | - echo "IDs: ${{ inputs.instance_id }}" - echo "AMI: ${{ inputs.instance_image_id }}" - echo "Type: ${{ inputs.instance_type }}" - echo "Request ID: ${{ inputs.request_id }}" - - - name: Get benchmark date - run: | - echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}" - - - name: Checkout tfhe-rs repo with tags - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Set up home - # "Install rust" step require root user to have a HOME directory which is not set. - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - - name: Install rust - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 - with: - toolchain: nightly - - - name: Run multi-bit benchmarks with AVX512 - run: | - make FAST_BENCH=TRUE bench_signed_integer_multi_bit - - - name: Parse benchmarks to csv - run: | - make PARSE_INTEGER_BENCH_CSV_FILE=${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} \ - parse_integer_benches - - - name: Upload csv results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_csv_integer - path: ${{ env.PARSE_INTEGER_BENCH_CSV_FILE }} - - - name: Parse results - run: | - COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" - COMMIT_HASH="$(git describe --tags --dirty)" - python3 ./ci/benchmark_parser.py target/criterion ${{ env.RESULTS_FILENAME }} \ - --database tfhe_rs \ - --hardware ${{ inputs.instance_type }} \ - --project-version "${COMMIT_HASH}" \ - --branch ${{ github.ref_name }} \ - --commit-date "${COMMIT_DATE}" \ - --bench-date "${{ env.BENCH_DATE }}" \ - --walk-subdirs \ - --name-suffix avx512 \ - --throughput - - - name: Upload parsed results artifact - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b - with: - name: ${{ github.sha }}_integer - path: ${{ env.RESULTS_FILENAME }} - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Send data to Slab - shell: bash - run: | - echo "Computing HMac on results file" - SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')" - echo "Sending results to Slab..." - curl -v -k \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: store_data_v2" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @${{ env.RESULTS_FILENAME }} \ - ${{ secrets.SLAB_URL }} - - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_MESSAGE: "Signed integer benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/start_benchmarks.yml b/.github/workflows/start_benchmarks.yml deleted file mode 100644 index 997c10ed14..0000000000 --- a/.github/workflows/start_benchmarks.yml +++ /dev/null @@ -1,123 +0,0 @@ -# Start all benchmark jobs on Slab CI bot. -name: Start all benchmarks - -on: - push: - branches: - - "main" - workflow_dispatch: - inputs: - # The input name must be the name of the slab command to launch - boolean_bench: - description: "Run Boolean benches" - type: boolean - default: true - shortint_bench: - description: "Run shortint benches" - type: boolean - default: true - integer_bench: - description: "Run integer benches" - type: boolean - default: true - signed_integer_bench: - description: "Run signed integer benches" - type: boolean - default: true - integer_multi_bit_bench: - description: "Run integer multi bit benches" - type: boolean - default: true - signed_integer_multi_bit_bench: - description: "Run signed integer multi bit benches" - type: boolean - default: true - core_crypto_bench: - description: "Run core crypto benches" - type: boolean - default: true - -jobs: - start-benchmarks: - if: ${{ (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs') || github.event_name == 'workflow_dispatch' }} - strategy: - matrix: - command: [ boolean_bench, shortint_bench, - integer_bench, integer_multi_bit_bench, - signed_integer_bench, signed_integer_multi_bit_bench, - core_crypto_bench ] - runs-on: ubuntu-latest - steps: - - name: Checkout tfhe-rs - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Check for file changes - id: changed-files - uses: tj-actions/changed-files@cc733854b1f224978ef800d29e4709d5ee2883e4 - with: - files_yaml: | - common_benches: - - toolchain.txt - - Makefile - - ci/slab.toml - - tfhe/Cargo.toml - - tfhe/src/core_crypto/** - - .github/workflows/start_benchmarks.yml - boolean_bench: - - tfhe/src/boolean/** - - tfhe/benches/boolean/** - - .github/workflows/boolean_benchmark.yml - shortint_bench: - - tfhe/src/shortint/** - - tfhe/benches/shortint/** - - .github/workflows/shortint_benchmark.yml - integer_bench: - - tfhe/src/shortint/** - - tfhe/src/integer/** - - tfhe/benches/integer/bench.rs - - .github/workflows/integer_benchmark.yml - integer_multi_bit_bench: - - tfhe/src/shortint/** - - tfhe/src/integer/** - - tfhe/benches/integer/bench.rs - - .github/workflows/integer_multi_bit_benchmark.yml - signed_integer_bench: - - tfhe/src/shortint/** - - tfhe/src/integer/** - - tfhe/benches/integer/signed_bench.rs - - .github/workflows/signed_integer_benchmark.yml - signed_integer_multi_bit_bench: - - tfhe/src/shortint/** - - tfhe/src/integer/** - - tfhe/benches/integer/signed_bench.rs - - .github/workflows/signed_integer_multi_bit_benchmark.yml - core_crypto_bench: - - tfhe/src/core_crypto/** - - tfhe/benches/core_crypto/** - - .github/workflows/core_crypto_benchmark.yml - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Start AWS job in Slab - # If manually triggered check that the current bench has been requested - # Otherwise if it's on push check that files relevant to benchmarks have changed - if: (github.event_name == 'workflow_dispatch' && github.event.inputs[matrix.command] == 'true') || (github.event_name == 'push' && (steps.changed-files.outputs.common_benches_any_changed == 'true' || steps.changed-files.outputs[format('{0}_any_changed', matrix.command)] == 'true')) - shell: bash - run: | - echo -n '{"command": "${{ matrix.command }}", "git_ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' > command.json - SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" - curl -v -k \ - --fail-with-body \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: start_aws" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @command.json \ - ${{ secrets.SLAB_URL }} diff --git a/.github/workflows/start_full_benchmarks.yml b/.github/workflows/start_full_benchmarks.yml deleted file mode 100644 index 41eabf3cfc..0000000000 --- a/.github/workflows/start_full_benchmarks.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Start all benchmark jobs, including full shortint and integer, on Slab CI bot. -name: Start full suite benchmarks - -on: - schedule: - # Weekly benchmarks will be triggered each Saturday at 1a.m. - - cron: '0 1 * * 6' - # Quarterly benchmarks will be triggered right before end of quarter, the 25th of the current month at 4a.m. - # These benchmarks are far longer to execute hence the reason to run them only four time a year. - - cron: '0 4 25 MAR,JUN,SEP,DEC *' - workflow_dispatch: - inputs: - benchmark_type: - description: 'Benchmark type' - required: true - default: 'weekly' - type: choice - options: - - weekly - - quarterly - -jobs: - start-benchmarks: - if: ${{ (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || github.event_name == 'workflow_dispatch' }} - strategy: - matrix: - command: [ boolean_bench, shortint_full_bench, - integer_full_bench, signed_integer_full_bench, - core_crypto_bench, wasm_client_bench ] - runs-on: ubuntu-latest - steps: - - name: Checkout tfhe-rs - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - fetch-depth: 0 - - - name: Checkout Slab repo - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - with: - repository: zama-ai/slab - path: slab - token: ${{ secrets.FHE_ACTIONS_TOKEN }} - - - name: Set benchmarks type as weekly - if: (github.event_name == 'workflow_dispatch' && inputs.benchmark_type == 'weekly') || github.event.schedule == '0 1 * * 6' - run: | - echo "BENCH_TYPE=weekly_benchmarks" >> "${GITHUB_ENV}" - - - name: Set benchmarks type as quarterly - if: (github.event_name == 'workflow_dispatch' && inputs.benchmark_type == 'quarterly') || github.event.schedule == '0 4 25 MAR,JUN,SEP,DEC *' - run: | - echo "BENCH_TYPE=quarterly_benchmarks" >> "${GITHUB_ENV}" - - - name: Start AWS job in Slab - shell: bash - run: | - echo -n '{"command": "${{ matrix.command }}", "git_ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "user_inputs": "${{ env.BENCH_TYPE }}"}' > command.json - SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')" - curl -v -k \ - --fail-with-body \ - -H "Content-Type: application/json" \ - -H "X-Slab-Repository: ${{ github.repository }}" \ - -H "X-Slab-Command: start_aws" \ - -H "X-Hub-Signature-256: sha256=${SIGNATURE}" \ - -d @command.json \ - ${{ secrets.SLAB_URL }}