[CI] support running simtest nightly on a particular commit (#16707)

## Description 

Otherwise people need to make new branches to test changes.

Another changes:
- Allow specifying `MSIM_TEST_NUM` per run.
- Increase time limit to 4hr.
- Do not notify if the run is not a scheduled run.

## Test Plan 


https://github.com/MystenLabs/sui/actions/runs/8311073333/job/22744289663

---
If your changes are not user-facing and do not break anything, you can
skip the following section. Otherwise, please briefly describe what has
changed under the Release Notes section.

### Type of Change (Check all that apply)

- [ ] protocol change
- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes

.github/workflows/simulator-nightly.yml

@@ -8,18 +8,24 @@ on:
     - cron: '0 9 * * *' # UTC timing is every day at 1am PST
   workflow_dispatch:
     inputs:
-      sui_commit:
-        description: "commit to test"
+      sui_ref:
+        description: "Branch / commit to test"
         type: string
         required: true
-        default: origin/main
+        default: main
+      test_num:
+        description: "MSIM_TEST_NUM (test iterations)"
+        type: string
+        required: false
+        default: "30"
 
 env:
-  SUI_COMMIT: "${{ github.event.inputs.sui_commit || 'origin/main' }}"
+  SUI_REF: "${{ github.event.inputs.sui_ref || 'main' }}"
+  TEST_NUM: "${{ github.event.inputs.test_num || '30' }}"
 
 jobs:
   simtest:
-    timeout-minutes: 120
+    timeout-minutes: 240
     permissions:
       # The "id-token: write" permission is required or Machine ID will not be
       # able to authenticate with the cluster.
@@ -59,7 +65,7 @@ jobs:
       # Checkout out the latest sui repo
       - name: Checkout sui repo
         run: |
-          tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git rebase ${{ env.SUI_COMMIT  }}"
+          tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git checkout ${{ env.SUI_REF }}"
 
       # Setting up cargo and simtest
       - name: Install simtest
@@ -69,13 +75,13 @@ jobs:
       # Run simulator tests
       - name: Run simtest
         run: |
-          tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 120 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 ./scripts/simtest/simtest-run.sh"
+            tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 120 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 MSIM_TEST_NUM=${{ env.TEST_NUM }} ./scripts/simtest/simtest-run.sh"
   
   notify:
     name: Notify
     needs: [simtest]
     runs-on: ubuntu-latest
-    if: failure()
+    if: github.event_name == 'schedule' && failure()
 
     steps:
     - uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # Pin v4.1.1

scripts/simtest/simtest-run.sh

@@ -33,12 +33,14 @@ SIMTEST_LOGS_DIR=~/simtest_logs
 LOG_DIR="${SIMTEST_LOGS_DIR}/${DATE}"
 LOG_FILE="$LOG_DIR/log"
 
+# By default run 30 iterations for each test. But allow overrides.
+${MSIM_TEST_NUM:=30}
+
 # This command runs many different tests, so it already uses all CPUs fairly efficiently, and
 # don't need to be done inside of the for loop below.
 # TODO: this logs directly to stdout since it is not being run in parallel. is that ok?
 MSIM_TEST_SEED="$SEED" \
 MSIM_WATCHDOG_TIMEOUT_MS=60000 \
-MSIM_TEST_NUM=30 \
 scripts/simtest/cargo-simtest simtest \
   --color always \
   --test-threads "$NUM_CPUS" \