devnet_release.yml

name: Devnet Release Scenarios

on:
  schedule:
    - cron: '0 */2 * * *'
  workflow_dispatch:

jobs:
  Run:
    strategy:
      fail-fast: false
      matrix:
        test:
        - FourValidatorsReconnect
        - MultipleValidatorsDown
        - FourValidatorsReconnectRmDatabase
        - FourValidatorsReconnectSpammer
        - MacroBlockProduction
        - Validators90sDown
        - LowViewChangeDelay
        - FiveValidatorsWithSpammer

        include:
        - test: FourValidatorsReconnect
          devnet_args: -R
        - test: MultipleValidatorsDown
          devnet_args: -k 2 -R
        - test: FourValidatorsReconnectRmDatabase
          devnet_args: -d -R
        - test: FourValidatorsReconnectSpammer
          devnet_args: -s 500 -R
        - test: MacroBlockProduction
          pre: "sed -i 's/BLOCKS_PER_BATCH: u32 = 32;/BLOCKS_PER_BATCH: u32 = 2;/g' primitives/src/policy.rs"
          devnet_args: -k 0 -s 150 -R
        - test: Validators90sDown
          devnet_args: -s 500 -t 90 -R
        - test: LowViewChangeDelay
          pre: "sed -i 's/VIEW_CHANGE_DELAY: Duration = Duration::from_secs(10);/VIEW_CHANGE_DELAY: Duration = Duration::from_secs(2);/g' validator/src/validator.rs"
          devnet_args: -k 0 -s 150 -R
        - test: FiveValidatorsWithSpammer
          devnet_args: --validators 5 -R -s 500 -k 0

    runs-on: ubuntu-20.04

    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v3
    - uses: actions/cache@v3
      with:
        path:
          ~/.cargo/registry
          ~/.cargo/git
          target
        key: cargo-${{ hashFiles('**/Cargo.toml') }}
    - name: Set up Rust toolchain
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: stable
        override: true
    - name: Optionally patch the source
      run: ${{ matrix.pre }}
    - uses: actions-rs/cargo@v1
      with:
        command: build
        args: --release
    - name: Retrieve initial timestamp
      id: initial_ts
      run: |
         echo "::set-output name=INITIAL_TS::$(date +%s%N | cut -b1-13)"
    - name: Execute the test
      run: |
          bash scripts/devnet/devnet.sh --run-environment ci ${{ matrix.devnet_args }}
      env:
        NIMIQ_LOKI_URL: ${{ secrets.LOKI_URL }}
        NIMIQ_LOKI_EXTRA_FIELDS: nimiq_ci_name=${{ github.workflow }}:nimiq_ci_run=${{ github.run_number }}:nimiq_test=${{ matrix.test }}
    - name: Redact Loki URL
      if: always()
      run: sed -i 's/url = "[^"]*"/url = "<redacted>"/g' temp-logs/*/conf/*/client.toml
    - name: Archive test results
      if: always()
      uses: actions/upload-artifact@v3
      with:
          name: ${{ matrix.test }}-logs
          path: |
            temp-logs/
    - name: Retrieve failure reason
      if: always()
      run: |
          if [ -f temp-state/RESULT.TXT ]; then
            echo "::set-output name=FAIL_REASON::$(cat temp-state/RESULT.TXT)"
          else
            echo "::set-output name=FAIL_REASON::other"
          fi
      id: reason
    - name: Retrieve final timestamp and Run ID
      if: always()
      id: final_ts_run_id
      run: |
         echo "::set-output name=FINAL_TS::$(date +%s%N | cut -b1-13)"
         echo "::set-output name=RUN_ID::$(cat temp-logs/*/conf/run_id.info)"
    - name: Build Grafana dashboard link
      if: always()
      id: grafana_url
      run: |
         echo "::set-output name=GRAFANA_URL::http://localhost:3001/d/YbjdvlU7z/nimiq-test?orgId=1&var-env=ci&var-run_id=${{steps.final_ts_run_id.outputs.RUN_ID}}&from=${{steps.initial_ts.outputs.INITIAL_TS}}&to=${{steps.final_ts_run_id.outputs.FINAL_TS}}"
    - name: Report potential deadlocks to slack
      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'DEADLOCK')
      uses: ravsamhq/notify-slack-action@v1
      with:
          status: ${{ job.status }}
          notification_title: 'Potential deadlock detected in ${{ matrix.test }}'
          footer: '<{run_url}|View Run>'
      env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
#    - name: Report long lock hold times
#      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'LONG_LOCK_HOLD_TIME')
#      uses: ravsamhq/notify-slack-action@v1
#      with:
#          status: ${{ job.status }}
#          notification_title: 'Long lock hold time in ${{ matrix.test }}'
#          footer: '<{run_url}|View Run>'
#      env:
#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
#    - name: Report slow lock acquisitions
#      if: always() && contains(steps.reason.outputs.FAIL_REASON, 'SLOW_LOCK_ACQUISITION')
#      uses: ravsamhq/notify-slack-action@v1
#      with:
#          status: ${{ job.status }}
#          notification_title: 'Slow lock acquisition detected in ${{ matrix.test }}'
#          footer: '<{run_url}|View Run>'
#      env:
#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
    - name: Report Status to Slack
      if: always() && github.ref == 'refs/heads/albatross'
      uses: ravsamhq/notify-slack-action@v1
      with:
          status: ${{ job.status }}
          notify_when: 'failure'
          notification_title: '${{ matrix.test }} failed because of ${{ steps.reason.outputs.FAIL_REASON }}'
          footer: '<{run_url}|View Run> | <${{steps.grafana_url.outputs.GRAFANA_URL}}|Grafana dashboard>'
      env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}