diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8793f1ee51..e2f3b31d86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,110 +293,15 @@ jobs: netsim-integration-tests: if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" - timeout-minutes: 30 - name: Run network simulations/benchmarks - runs-on: [self-hosted, linux, X64] - env: - # Using self-hosted runners so use local cache for sccache and - # not SCCACHE_GHA_ENABLED. - RUSTC_WRAPPER: "sccache" - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install rust stable - uses: dtolnay/rust-toolchain@stable - - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.5 - - - name: Build iroh - run: | - cargo build --release --workspace --all-features - - - name: Fetch and build chuck - run: | - cd .. - rm -rf chuck - git clone https://github.com/n0-computer/chuck.git - cd chuck - cargo build --release - - - name: Install netsim deps - run: | - cd ../chuck/netsim - sudo apt update - ./setup.sh - - - name: Copy binaries to right location - run: | - cp target/release/iroh ../chuck/netsim/bins/iroh - cp target/release/iroh-relay ../chuck/netsim/bins/iroh-relay - cp ../chuck/target/release/chuck ../chuck/netsim/bins/chuck - - - name: Run tests - run: | - cd ../chuck/netsim - sudo kill -9 $(pgrep ovs) || true - sudo mn --clean || true - sudo python3 main.py ${{ runner.debug && '--debug' || ''}} --integration sims/iroh/iroh.json - sudo python3 main.py ${{ runner.debug && '--debug' || ''}} --integration sims/integration - - name: Cleanup - run: | - sudo kill -9 $(pgrep iroh-relay) || true - sudo kill -9 $(pgrep iroh) || true - sudo kill -9 $(pgrep ovs) || true - sudo mn --clean || true - - - name: Setup Environment (PR) - if: ${{ github.event_name == 'pull_request' }} - shell: bash - run: | - echo "LAST_COMMIT_SHA=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})" >> ${GITHUB_ENV} - echo "HEAD_REF=${{ github.event.pull_request.head.ref }}" >> ${GITHUB_ENV} - - name: Setup Environment (Push) - if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} - shell: bash - run: | - echo "LAST_COMMIT_SHA=$(git rev-parse --short ${GITHUB_SHA})" >> ${GITHUB_ENV} - echo "HEAD_REF=${{ github.head_ref }}" >> ${GITHUB_ENV} - - - name: Generate reports - run: | - cd ../chuck/netsim - python3 reports_csv.py --prom --commit ${{ env.LAST_COMMIT_SHA }} > report_prom.txt - python3 reports_csv.py --metro --commit ${{ env.LAST_COMMIT_SHA }} > report_metro.txt - python3 reports_csv.py --metro --integration --commit ${{ env.LAST_COMMIT_SHA }} > report_metro_integration.txt - - - name: Dump report - if: ${{ (github.event.pull_request && !github.event.pull_request.head.repo.fork) || !github.event.pull_request }} - run: | - export AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}} - export AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}} - export AWS_DEFAULT_REGION=us-west-2 - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - sudo ./aws/install --update - - cd ../chuck/netsim - - aws_fname=intg_${{ env.LAST_COMMIT_SHA }}.tar.gz - tar cvzf report.tar.gz report_prom.txt report_metro.txt report_metro_integration.txt logs/ report/ viz/ - aws s3 cp ./report.tar.gz s3://${{secrets.S3_REPORT_BUCKET}}/$aws_fname --no-progress - - - name: Echo metrics - run: | - cd ../chuck/netsim - d=$(cat report_metro.txt) - metro_data=$(printf "%s\n " "$d") - echo "$metro_data" - d=$(cat report_metro_integration.txt) - metro_data=$(printf "%s\n " "$d") - echo "$metro_data" - + uses: './.github/workflows/netsim_runner.yaml' + secrets: inherit + with: + branch: ${{ github.ref }} + max_workers: 4 + netsim_branch: "main" + sim_paths: "sims/iroh/iroh.json,sims/integration" + pr_number: ${{ github.event.pull_request.number || '' }} + docker_build_and_test: name: Docker Test if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" diff --git a/.github/workflows/netsim.yml b/.github/workflows/netsim.yml index 49235d0a86..2e57ab8ac4 100644 --- a/.github/workflows/netsim.yml +++ b/.github/workflows/netsim.yml @@ -4,8 +4,22 @@ on: push: branches: - main - issue_comment: - types: [created, edited, deleted] + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + branch: + required: true + type: string + netsim_branch: + required: true + type: string + default: "main" + report_table: + required: false + type: boolean + default: false env: RUST_BACKTRACE: 1 @@ -16,213 +30,29 @@ env: IROH_FORCE_STAGING_RELAYS: "1" jobs: - netsim: - timeout-minutes: 60 - name: Run network simulations/benchmarks - if: >- - (github.event_name == 'issue_comment' && - github.event.issue.pull_request && - startsWith(github.event.comment.body, '/netsim')) || github.event_name != 'issue_comment' - runs-on: [self-hosted, linux, X64] - permissions: - issues: write - pull-requests: write - steps: - - name: Checkout - if: github.event_name != 'issue_comment' - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Resolve PR refs (Issue) - if: ${{ github.event_name == 'issue_comment' }} - id: refs - uses: arqu/resolve-pr-refs@main - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Checkout - if: github.event_name == 'issue_comment' - uses: actions/checkout@v4 - with: - repository: ${{ github.event.issue.repository.full_name }} - submodules: recursive - ref: ${{ steps.refs.outputs.head_ref }} - - - name: Install rust stable - uses: dtolnay/rust-toolchain@stable - - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.5 - - - name: Build iroh - run: | - cargo build --profile optimized-release --workspace --all-features - - - name: Detect chuck branch - if: github.event_name == 'issue_comment' - id: detect_chuck_branch - shell: bash - run: | - a='${{ github.event.comment.body }}' - if [[ $a == '/netsim branch'* ]]; - then - a=$(echo "$a" | tr '\n' ' ' | tr -s " " | cut -d ' ' -f3) - branch="$a" - echo "CHUCK_BRANCH=$branch" >> ${GITHUB_ENV} - else - echo "CHUCK_BRANCH=main" >> ${GITHUB_ENV} - fi - - - name: Detect chuck branch if not comment - if: github.event_name != 'issue_comment' - shell: bash - run: | - echo "CHUCK_BRANCH=main" >> ${GITHUB_ENV} - - - name: Fetch and build chuck - shell: bash - run: | - cd .. - rm -rf chuck - git clone --single-branch --branch ${{ env.CHUCK_BRANCH }} https://github.com/n0-computer/chuck.git - cd chuck - cargo build --release - - - name: Install netsim deps - run: | - cd ../chuck/netsim - sudo apt update - ./setup.sh - - - name: Copy binaries to right location - run: | - cp target/optimized-release/iroh ../chuck/netsim/bins/iroh - cp target/optimized-release/iroh-relay ../chuck/netsim/bins/iroh-relay - cp ../chuck/target/release/chuck ../chuck/netsim/bins/chuck - - - name: Detect comment commands - if: github.event_name == 'issue_comment' - id: detect_comment_config - run: | - a='${{ github.event.comment.body }}' - if [[ $a == '/netsim config'* ]]; - then - a=$(echo "'$a'" | tr '\n' ' ' | tr -s " " | sed -e 's/.*```\(.*\)```.*/\1/') - export NETSIM_CONFIG="$a" - echo "NETSIM_CONFIG<> ${GITHUB_OUTPUT} - echo "${NETSIM_CONFIG}" >> ${GITHUB_OUTPUT} - echo "EOFMARKER" >> ${GITHUB_OUTPUT} - fi - echo "'$NETSIM_CONFIG'" - - - name: Run tests - run: | - cd ../chuck/netsim - sudo kill -9 $(pgrep ovs) || true - sudo mn --clean || true - c='${{ steps.detect_comment_config.outputs.NETSIM_CONFIG }}' - if [ -z "${c}" ]; - then - sudo python3 main.py --integration sims/iroh - sudo python3 main.py --integration sims/integration - else - echo $c >> custom_sim.json - sudo python3 main.py custom_sim.json - fi - - - name: Cleanup - run: | - sudo kill -9 $(pgrep iroh-relay) || true - sudo kill -9 $(pgrep iroh) || true - sudo kill -9 $(pgrep ovs) || true - sudo mn --clean || true - - - name: Generate report - id: generate_report - run: | - cd ../chuck/netsim - python3 reports_csv.py --table > report.txt - export NETSIM_REPORT=$(cat report.txt) - echo "NETSIM_REPORT<> ${GITHUB_OUTPUT} - echo "${NETSIM_REPORT}" >> ${GITHUB_OUTPUT} - echo "EOFMARKER" >> ${GITHUB_OUTPUT} - - - name: Setup Environment - shell: bash - run: | - echo "LAST_COMMIT_SHA=unknown" >> ${GITHUB_ENV} - echo "HEAD_REF=unknown" >> ${GITHUB_ENV} - - - name: Setup Environment (Issue) - if: ${{ github.event_name == 'issue_comment' }} - shell: bash - run: | - echo "LAST_COMMIT_SHA=${{ steps.refs.outputs.head_sha }}" >> ${GITHUB_ENV} - echo "HEAD_REF=${{ steps.refs.outputs.head_ref }}" >> ${GITHUB_ENV} - - name: Setup Environment (PR) - if: ${{ github.event_name == 'pull_request' }} - shell: bash - run: | - echo "LAST_COMMIT_SHA=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})" >> ${GITHUB_ENV} - echo "HEAD_REF=${{ github.event.pull_request.head.ref }}" >> ${GITHUB_ENV} - - name: Setup Environment (Push) - if: ${{ github.event_name == 'push' }} - shell: bash - run: | - echo "LAST_COMMIT_SHA=$(git rev-parse --short ${GITHUB_SHA})" >> ${GITHUB_ENV} - echo "HEAD_REF=${{ github.ref }}" >> ${GITHUB_ENV} - - - name: Respond Issue - uses: peter-evans/create-or-update-comment@v4 - if: github.event_name == 'issue_comment' - with: - issue-number: ${{ github.event.issue.number }} - body: | - `${{ env.HEAD_REF }}.${{ env.LAST_COMMIT_SHA }}` - Perf report: - ${{ steps.generate_report.outputs.NETSIM_REPORT }} - - - name: Respond PR - uses: peter-evans/create-or-update-comment@v4 - if: github.event.pull_request - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - `${{ env.HEAD_REF }}.${{ env.LAST_COMMIT_SHA }}` - Perf report: - ${{ steps.generate_report.outputs.NETSIM_REPORT }} - - - name: Generate reports - run: | - cd ../chuck/netsim - python3 reports_csv.py --prom --commit ${{ env.LAST_COMMIT_SHA }} > report_prom.txt - python3 reports_csv.py --metro --commit ${{ env.LAST_COMMIT_SHA }} > report_metro.txt - python3 reports_csv.py --metro --integration --commit ${{ env.LAST_COMMIT_SHA }} > report_metro_integration.txt - - - name: Dump report - run: | - export AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}} - export AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}} - export AWS_DEFAULT_REGION=us-west-2 - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip -q awscliv2.zip - sudo ./aws/install --update - - cd ../chuck/netsim - - aws_fname=${{ env.LAST_COMMIT_SHA }}.tar.gz - tar cvzf report.tar.gz report_prom.txt report.txt report_metro.txt report_metro_integration.txt logs/ report/ viz/ - aws s3 cp ./report.tar.gz s3://${{secrets.S3_REPORT_BUCKET}}/$aws_fname --no-progress - - - name: Post metrics - if: github.ref_name=='main' && github.event_name != 'issue_comment' - run: | - cd ../chuck/netsim - d=$(cat report_metro.txt) - metro_data=$(printf "%s\n " "$d") - curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${{secrets.METRO_TOKEN}}" --data "$metro_data" ${{secrets.METRO_ENDPOINT}} - d=$(cat report_metro_integration.txt) - metro_data=$(printf "%s\n " "$d") - curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${{secrets.METRO_TOKEN}}" --data "$metro_data" ${{secrets.METRO_ENDPOINT}} + netsim-release: + if: ${{github.ref_name=='main' && github.event_name == 'push'}} + uses: './.github/workflows/netsim_runner.yaml' + secrets: inherit + with: + branch: "main" + max_workers: 1 + netsim_branch: "main" + sim_paths: "sims/iroh,sims/integration" + pr_number: "" + publish_metrics: true + build_profile: "optimized-release" + + netsim-perf: + if: ${{github.event_name != 'push'}} + uses: './.github/workflows/netsim_runner.yaml' + secrets: inherit + with: + branch: ${{inputs.branch}} + max_workers: 1 + netsim_branch: ${{inputs.netsim_branch}} + sim_paths: "sims/iroh" + pr_number: ${{inputs.pr_number}} + publish_metrics: false + build_profile: "optimized-release" + report_table: ${{inputs.report_table}} diff --git a/.github/workflows/netsim_runner.yaml b/.github/workflows/netsim_runner.yaml new file mode 100644 index 0000000000..138dac54c1 --- /dev/null +++ b/.github/workflows/netsim_runner.yaml @@ -0,0 +1,255 @@ +name: netsim-runner + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + max_workers: + required: true + type: number + default: 4 + netsim_branch: + required: true + type: string + default: "main" + debug_logs: + required: false + type: boolean + default: false + build_profile: + required: false + type: string + default: "release" + sim_paths: + required: false + type: string + default: "sims/iroh,sims/integration" + publish_metrics: + required: false + type: boolean + default: false + visualizations: + required: false + type: boolean + default: false + pr_number: + required: false + type: string + default: "" + report_table: + required: false + type: boolean + default: false + workflow_call: + inputs: + branch: + required: true + type: string + max_workers: + required: true + type: number + default: 4 + netsim_branch: + required: true + type: string + default: "main" + debug_logs: + required: false + type: boolean + default: false + build_profile: + required: false + type: string + default: "release" + sim_paths: + required: false + type: string + default: "sims/iroh,sims/integration" + publish_metrics: + required: false + type: boolean + default: false + visualizations: + required: false + type: boolean + default: false + pr_number: + required: false + type: string + default: "" + report_table: + required: false + type: boolean + default: false + +env: + RUST_BACKTRACE: 1 + RUSTFLAGS: -Dwarnings + MSRV: "1.66" + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + IROH_FORCE_STAGING_RELAYS: "1" + +jobs: + netsim: + name: Netsim + timeout-minutes: 45 + runs-on: [self-hosted, linux, X64] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ inputs.branch }} + + - name: Install rust stable + uses: dtolnay/rust-toolchain@stable + + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.5 + + - name: Build iroh + run: | + cargo build --profile ${{ inputs.build_profile }} --workspace --all-features + + - name: Fetch and build chuck + run: | + cd .. + rm -rf chuck + git clone --single-branch --branch ${{ inputs.netsim_branch }} https://github.com/n0-computer/chuck.git + cd chuck + cargo build --release + + - name: Install netsim deps + run: | + cd ../chuck/netsim + sudo apt update + ./setup.sh + ./cleanup.sh || true + + - name: Copy binaries to right location + run: | + cp target/${{inputs.build_profile}}/iroh ../chuck/netsim/bins/iroh + cp target/${{inputs.build_profile}}/iroh-relay ../chuck/netsim/bins/iroh-relay + cp ../chuck/target/release/chuck ../chuck/netsim/bins/chuck + + - name: Get commit sha + shell: bash + run: | + echo "LAST_COMMIT_SHA=$(git rev-parse --short ${GITHUB_SHA})" >> ${GITHUB_ENV} + + - name: Run tests + run: | + cd ../chuck/netsim + # split sim_paths by comma + IFS=',' read -ra sim_paths <<< "${{ inputs.sim_paths }}" + for sim_path in "${sim_paths[@]}"; do + sudo python3 main.py ${{ inputs.debug_logs && '--debug' || ''}} ${{ inputs.visualizations && '--visualize' || ''}} --max-workers=${{ inputs.max_workers }} --integration $sim_path + done + + - name: Generate report + id: generate_report + run: | + cd ../chuck/netsim + python3 reports_csv.py --table > report_table.txt + python3 reports_csv.py --prom --commit ${{ env.LAST_COMMIT_SHA }} > report_prom.txt + python3 reports_csv.py --metro --commit ${{ env.LAST_COMMIT_SHA }} > report_metro.txt + python3 reports_csv.py --metro --integration --commit ${{ env.LAST_COMMIT_SHA }} > report_metro_integration.txt + + - name: Dump report + run: | + export AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}} + export AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}} + export AWS_DEFAULT_REGION=us-west-2 + + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + sudo ./aws/install --update + + cd ../chuck/netsim + + aws_fname=${{ env.LAST_COMMIT_SHA }}.tar.gz + tar -cvzf report.tar.gz report_prom.txt report_table.txt report_metro.txt report_metro_integration.txt logs/ report/ viz/ + aws s3 cp ./report.tar.gz s3://${{secrets.S3_REPORT_BUCKET}}/$aws_fname --no-progress + + - name: Move report + run: | + cp ../chuck/netsim/report.tar.gz ./report.tar.gz + + - name: Upload report + uses: actions/upload-artifact@v4 + id: upload-report + with: + name: netsim-report-${{ env.LAST_COMMIT_SHA }} + path: report.tar.gz + retention-days: 3 + overwrite: true + + - name: Find Docs Comment + if: ${{ inputs.pr_number != '' }} + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ inputs.pr_number }} + comment-author: 'github-actions[bot]' + body-includes: Netsim report for this PR has been generated + + - name: Create or Update Docs Comment + if: ${{ inputs.pr_number != '' }} + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.fc.outputs.comment-id }} + body: | + Netsim report & logs for this PR have been generated and is available at: [LOGS](${{steps.upload-report.outputs.artifact-url}}) + This report will remain available for 3 days. + + Last updated for commit: ${{ env.LAST_COMMIT_SHA }} + edit-mode: replace + + - name: Generate report table + if: ${{ inputs.pr_number != '' && inputs.report_table}} + id: generate_report_table + run: | + cd ../chuck/netsim + export NETSIM_REPORT=$(cat report_table.txt) + echo "NETSIM_REPORT<> ${GITHUB_OUTPUT} + echo "${NETSIM_REPORT}" >> ${GITHUB_OUTPUT} + echo "EOFMARKER" >> ${GITHUB_OUTPUT} + + - name: Echo Report Table on PR + uses: peter-evans/create-or-update-comment@v4 + if: ${{ inputs.pr_number != '' && inputs.report_table }} + with: + issue-number: ${{ inputs.pr_number }} + body: | + `${{ inputs.branch }}.${{ env.LAST_COMMIT_SHA }}` + Perf report: + ${{ steps.generate_report_table.outputs.NETSIM_REPORT }} + + - name: Publish metrics + if: ${{ inputs.publish_metrics }} + run: | + cd ../chuck/netsim + d=$(cat report_metro.txt) + metro_data=$(printf "%s\n " "$d") + curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${{secrets.METRO_TOKEN}}" --data "$metro_data" ${{secrets.METRO_ENDPOINT}} + d=$(cat report_metro_integration.txt) + metro_data=$(printf "%s\n " "$d") + curl -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${{secrets.METRO_TOKEN}}" --data "$metro_data" ${{secrets.METRO_ENDPOINT}} + + - name: Echo metrics (debug) + run: | + cd ../chuck/netsim + d=$(cat report_metro.txt) + metro_data=$(printf "%s\n " "$d") + echo "$metro_data" + d=$(cat report_metro_integration.txt) + metro_data=$(printf "%s\n " "$d") + echo "$metro_data" + + - name: Cleanup + run: | + ./cleanup.sh || true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f86e97686..40412157bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -288,5 +288,4 @@ jobs: with: release_version: ${{ needs.build_release.outputs.release_version }} base_hash: ${{ needs.build_release.outputs.base_hash }} - publish: true - \ No newline at end of file + publish: true \ No newline at end of file