fix nightly simtests (#4963)

* fix(tests): make nightly simtests deterministic by default * fix what looked like an error * prefix vars * fix: comment unstable objects to fix `basic_read_cmd_snapshot_tests` with random seeds * revert: remove `SIMTEST_USE_DATE_AS_SEED` because we want it seeded by default * rust fmt --------- Co-authored-by: muXxer <[email protected]>
iotaledger · Jan 23, 2025 · 7233468 · 7233468
1 parent 98552b3
commit 7233468
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 61 deletions.
diff --git a/crates/iota-e2e-tests/tests/snapshot_tests.rs b/crates/iota-e2e-tests/tests/snapshot_tests.rs
@@ -63,8 +63,13 @@ async fn basic_read_cmd_snapshot_tests() -> Result<(), anyhow::Error> {
         "iota client objects 0x0000000000000000000000000000000000000000000000000000000000000000", /* empty addr */
         "iota client object 0x5",       // valid object
         "iota client object 0x5 --bcs", // valid object BCS
-        "iota client object 0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3", /* valid object */
-        "iota client object 0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3 --bcs", /* valid object BCS */
+        // Simtest object IDs are not stable so these object IDs may or may not exist currently --
+        // commenting them out for now.
+        // WARNING: Do not uncomment this, because that will break simtests with random
+        // MSIM_TEST_SEED! "iota client object
+        // 0x3b5121a0603ef7ab4cb57827fceca17db3338ef2cd76126cc1523b681df27cee", // valid object
+        // "iota client object 0x3b5121a0603ef7ab4cb57827fceca17db3338ef2cd76126cc1523b681df27cee
+        // --bcs", // valid object BCS
         "iota client object 0x0000000000000000000000000000000000000000000000000000000000000000", /* non-existent object */
         "iota client tx-block 88FqW2hyUgShTyLcGzbh6scZB45XZYmXpXSxydkBVTPu", // valid tx digest
         "iota client tx-block 11111111111111111111111111111111",             /* non-existent tx

diff --git a/crates/iota-e2e-tests/tests/snapshots/snapshot_tests__body_fn.snap b/crates/iota-e2e-tests/tests/snapshots/snapshot_tests__body_fn.snap
@@ -1,6 +1,7 @@
 ---
 source: crates/iota-e2e-tests/tests/snapshot_tests.rs
 expression: "run_one(cmds, context).await?"
+snapshot_kind: text
 ---
 [
   "iota client objects {ME}",
@@ -175,54 +176,6 @@ expression: "run_one(cmds, context).await?"
       }
     }
   ],
-  "iota client object 0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3",
-  [
-    {
-      "data": {
-        "objectId": "0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3",
-        "version": "1",
-        "digest": "6cjAEhKn8mmgQC6TPSrFASBNUBcSkTscuYpVdBEBbbfz",
-        "type": "0x2::coin::Coin<0x2::iota::IOTA>",
-        "owner": {
-          "AddressOwner": "0x2868fed4dbeb23d2ace3ee3ad6e39061423c5692a2b289b39c643c0baf2d8d85"
-        },
-        "previousTransaction": "88FqW2hyUgShTyLcGzbh6scZB45XZYmXpXSxydkBVTPu",
-        "storageRebate": "0",
-        "content": {
-          "dataType": "moveObject",
-          "type": "0x2::coin::Coin<0x2::iota::IOTA>",
-          "fields": {
-            "balance": "30000000000000000",
-            "id": {
-              "id": "0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3"
-            }
-          }
-        }
-      }
-    }
-  ],
-  "iota client object 0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3 --bcs",
-  [
-    {
-      "data": {
-        "objectId": "0x4d03f39deb5e27a76a568adb591da553688e6df6fb053bc9ac069f2bd9495ae3",
-        "version": "1",
-        "digest": "6cjAEhKn8mmgQC6TPSrFASBNUBcSkTscuYpVdBEBbbfz",
-        "type": "0x2::coin::Coin<0x2::iota::IOTA>",
-        "owner": {
-          "AddressOwner": "0x2868fed4dbeb23d2ace3ee3ad6e39061423c5692a2b289b39c643c0baf2d8d85"
-        },
-        "previousTransaction": "88FqW2hyUgShTyLcGzbh6scZB45XZYmXpXSxydkBVTPu",
-        "storageRebate": "0",
-        "bcs": {
-          "dataType": "moveObject",
-          "type": "0x2::coin::Coin<0x2::iota::IOTA>",
-          "version": 1,
-          "bcsBytes": "TQPzneteJ6dqVorbWR2lU2iObfb7BTvJrAafK9lJWuMAAENP15RqAA=="
-        }
-      }
-    }
-  ],
   "iota client object 0x0000000000000000000000000000000000000000000000000000000000000000",
   [
     {

diff --git a/scripts/simtest/simtest-run.sh b/scripts/simtest/simtest-run.sh
@@ -3,7 +3,14 @@
 # Modifications Copyright (c) 2024 IOTA Stiftung
 # SPDX-License-Identifier: Apache-2.0
 
+# INPUTS
+# If tests timeout on your machine, override the per test timeout:
+SIMTEST_PER_TEST_TIMEOUT_MS=${SIMTEST_PER_TEST_TIMEOUT_MS:-60000}
+# Override the default dir for logs output:
+SIMTEST_LOGS_DIR="${SIMTEST_LOGS_DIR:-"$HOME/simtest_logs"}"
+
 echo "Running simulator tests at commit $(git rev-parse HEAD)"
+echo "Using SIMTEST_PER_TEST_TIMEOUT_MS=${SIMTEST_PER_TEST_TIMEOUT_MS} from env var"
 
 # Function to handle SIGINT signal (Ctrl+C)
 cleanup() {
@@ -12,28 +19,29 @@ cleanup() {
     kill -- "-$$"
     exit 1
 }
-
 # Set up the signal handler
 trap cleanup SIGINT
 
 if [ -z "$NUM_CPUS" ]; then
-  NUM_CPUS=$(cat /proc/cpuinfo | grep processor | wc -l) # ubuntu
+  if [ "$(uname -s)" == "Darwin" ]; 
+    then NUM_CPUS="$(sysctl -n hw.ncpu)"; # mac
+    else NUM_CPUS=$(cat /proc/cpuinfo | grep processor | wc -l) # ubuntu
+  fi
 fi
 
 # filter out some tests that give spurious failures.
 TEST_FILTER="(not (test(~batch_verification_tests)))"
 
+# we seed the rng with the current date
 DATE=$(date +%s)
 SEED="$DATE"
 
-# create logs directory
-SIMTEST_LOGS_DIR=~/simtest_logs
-[ ! -d ${SIMTEST_LOGS_DIR} ] && mkdir -p ${SIMTEST_LOGS_DIR}
-[ ! -d ${SIMTEST_LOGS_DIR}/${DATE} ] && mkdir -p ${SIMTEST_LOGS_DIR}/${DATE}
-
 LOG_DIR="${SIMTEST_LOGS_DIR}/${DATE}"
 LOG_FILE="$LOG_DIR/log"
 
+# create the log directory if it doesn't exist
+mkdir -p "$LOG_DIR"
+
 # By default run 1 iteration for each test, if not specified.
 : ${TEST_NUM:=1}
 
@@ -48,7 +56,7 @@ date
 # TODO: this logs directly to stdout since it is not being run in parallel. is that ok?
 MSIM_TEST_SEED="$SEED" \
 MSIM_TEST_NUM=${TEST_NUM} \
-MSIM_WATCHDOG_TIMEOUT_MS=60000 \
+MSIM_WATCHDOG_TIMEOUT_MS=${SIMTEST_PER_TEST_TIMEOUT_MS} \
 scripts/simtest/cargo-simtest simtest \
   --color always \
   --test-threads "$NUM_CPUS" \
@@ -72,7 +80,7 @@ for SUB_SEED in `seq 1 $NUM_CPUS`; do
   # --test-threads 1 is important: parallelism is achieved via the for loop
   MSIM_TEST_SEED="$SEED" \
   MSIM_TEST_NUM=1 \
-  MSIM_WATCHDOG_TIMEOUT_MS=60000 \
+  MSIM_WATCHDOG_TIMEOUT_MS=${SIMTEST_PER_TEST_TIMEOUT_MS} \
   SIM_STRESS_TEST_DURATION_SECS=300 \
   scripts/simtest/cargo-simtest simtest \
     --color always \
@@ -98,8 +106,8 @@ echo "Using MSIM_TEST_SEED=$SEED, logging to $LOG_FILE"
 
 MSIM_TEST_SEED="$SEED" \
 MSIM_TEST_NUM=1 \
-MSIM_WATCHDOG_TIMEOUT_MS=60000 \
-MSIM_TEST_CHECK_DETERMINISM=1
+MSIM_WATCHDOG_TIMEOUT_MS=${SIMTEST_PER_TEST_TIMEOUT_MS} \
+MSIM_TEST_CHECK_DETERMINISM=1 \
 scripts/simtest/cargo-simtest simtest \
   --color always \
   --test-threads "$NUM_CPUS" \