feat: run sealevel E2E in parallel with EVM

 - separate evm e2e tests with sealevel e2e tests
  - refactor relative paths code to use more absolute paths
  - update working directory code to be more reliable
 - update github CI to run these 2 tests in parallel

.github/workflows/test.yml

@@ -211,7 +211,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-type: [cosmwasm, non-cosmwasm]
+        e2e-type: [cosmwasm, evm, sealevel]
     steps:
       - uses: actions/setup-node@v4
         with:
@@ -238,7 +238,7 @@ jobs:
           save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
           workspaces: |
             ./rust/main
-            ${{ matrix.e2e-type == 'non-cosmwasm' && './rust/sealevel' || '' }}
+            ${{ matrix.e2e-type != 'cosmwasm' && './rust/sealevel' || '' }}
 
       - name: Free disk space
         run: |
@@ -268,7 +268,7 @@ jobs:
         uses: ./.github/actions/checkout-registry
 
       - name: agent tests (CosmWasm)
-        run: cargo test --release --package run-locally --bin run-locally --features cosmos test-utils -- cosmos::test --nocapture
+        run: cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture
         if: matrix.e2e-type == 'cosmwasm'
         working-directory: ./rust/main
         env:
@@ -284,16 +284,25 @@ jobs:
             echo "rust_changes=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: agent tests (EVM and Sealevel)
+      - name: agent tests (EVM)
         run: cargo run --release --bin run-locally --features test-utils
-        if: matrix.e2e-type == 'non-cosmwasm'
+        if: matrix.e2e-type == 'evm'
+        working-directory: ./rust/main
+        env:
+          E2E_CI_MODE: 'true'
+          E2E_CI_TIMEOUT_SEC: '600'
+          E2E_KATHY_MESSAGES: '20'
+          RUST_BACKTRACE: 'full'
+
+      - name: agent tests (Sealevel)
+        run: cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture
+        if: matrix.e2e-type == 'evm' && ${{ steps.check-rust-changes.outputs.rust_changes }}
         working-directory: ./rust/main
         env:
           E2E_CI_MODE: 'true'
           E2E_CI_TIMEOUT_SEC: '600'
           E2E_KATHY_MESSAGES: '20'
           RUST_BACKTRACE: 'full'
-          SEALEVEL_ENABLED: ${{ steps.check-rust-changes.outputs.rust_changes }}
 
   env-test:
     runs-on: ubuntu-latest
@@ -367,4 +376,4 @@ jobs:
       - name: Upload coverage reports to Codecov with GitHub Action
         uses: codecov/codecov-action@v4
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}

rust/README.md

@@ -112,10 +112,18 @@ validator. By default, this test will run indefinitely, but can be stopped with
 
 To run the tests for a specific VM, use the `--features` flag.
 
+##### Cosmos E2E Test
+
 ```bash
 cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture
 ```
 
+##### Sealevel E2E Test
+
+```bash
+cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture
+```
+
 ### Building Agent Docker Images
 
 There exists a docker build for the agent binaries. These docker images are used for deploying the agents in a

rust/main/utils/run-locally/Cargo.toml

@@ -43,3 +43,4 @@ vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] }
 
 [features]
 cosmos = []
+sealevel = []

rust/main/utils/run-locally/src/config.rs

@@ -7,7 +7,6 @@ pub struct Config {
     pub ci_mode: bool,
     pub ci_mode_timeout: u64,
     pub kathy_messages: u64,
-    pub sealevel_enabled: bool,
     // TODO: Include count of sealevel messages in a field separate from `kathy_messages`?
 }
 
@@ -28,9 +27,6 @@ impl Config {
                     .map(|r| r.parse::<u64>().unwrap());
                 r.unwrap_or(16)
             },
-            sealevel_enabled: env::var("SEALEVEL_ENABLED")
-                .map(|k| k.parse::<bool>().unwrap())
-                .unwrap_or(true),
         })
     }
 }

rust/main/utils/run-locally/src/cosmos/mod.rs

@@ -30,7 +30,9 @@ use crate::cosmos::link::link_networks;
 use crate::logging::log;
 use crate::metrics::agent_balance_sum;
 use crate::program::Program;
-use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle};
+use crate::utils::{
+    as_task, concat_path, get_workspace_path, stop_child, AgentHandles, TaskHandle,
+};
 use crate::{fetch_metric, AGENT_BIN_PATH};
 use cli::{OsmosisCLI, OsmosisEndpoint};
 
@@ -345,10 +347,12 @@ fn run_locally() {
     const TIMEOUT_SECS: u64 = 60 * 10;
     let debug = false;
 
+    let workspace_path = get_workspace_path();
+
     log!("Building rust...");
     Program::new("cargo")
         .cmd("build")
-        .working_dir("../../")
+        .working_dir(&workspace_path)
         .arg("features", "test-utils")
         .arg("bin", "relayer")
         .arg("bin", "validator")
@@ -529,7 +533,8 @@ fn run_locally() {
     // give things a chance to fully start.
     sleep(Duration::from_secs(10));
 
-    let starting_relayer_balance: f64 = agent_balance_sum(hpl_rly_metrics_port).unwrap();
+    let starting_relayer_balance: f64 =
+        agent_balance_sum(hpl_rly_metrics_port).expect("Failed to get relayer agent balance");
 
     // dispatch the second batch of messages (after agents start)
     dispatched_messages += dispatch(&osmosisd, linker, &nodes);
@@ -664,7 +669,8 @@ fn termination_invariants_met(
         return Ok(false);
     }
 
-    let ending_relayer_balance: f64 = agent_balance_sum(relayer_metrics_port).unwrap();
+    let ending_relayer_balance: f64 =
+        agent_balance_sum(relayer_metrics_port).expect("Failed to get relayer agent balance");
 
     // Make sure the balance was correctly updated in the metrics.
     // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here:

rust/main/utils/run-locally/src/ethereum/mod.rs

@@ -10,15 +10,18 @@ use crate::config::Config;
 use crate::ethereum::multicall::{DEPLOYER_ADDRESS, SIGNED_DEPLOY_MULTICALL_TX};
 use crate::logging::log;
 use crate::program::Program;
-use crate::utils::{as_task, AgentHandles, TaskHandle};
-use crate::{INFRA_PATH, MONOREPO_ROOT_PATH};
+use crate::utils::{as_task, get_ts_infra_path, get_workspace_path, AgentHandles, TaskHandle};
 
 mod multicall;
 
 #[apply(as_task)]
 pub fn start_anvil(config: Arc<Config>) -> AgentHandles {
     log!("Installing typescript dependencies...");
-    let yarn_monorepo = Program::new("yarn").working_dir(MONOREPO_ROOT_PATH);
+
+    let workspace_path = get_workspace_path();
+    let ts_infra_path = get_ts_infra_path(&workspace_path);
+
+    let yarn_monorepo = Program::new("yarn").working_dir(workspace_path);
     if !config.is_ci_env {
         // test.yaml workflow installs dependencies
         yarn_monorepo.clone().cmd("install").run().join();
@@ -42,7 +45,7 @@ pub fn start_anvil(config: Arc<Config>) -> AgentHandles {
 
     sleep(Duration::from_secs(10));
 
-    let yarn_infra = Program::new("yarn").working_dir(INFRA_PATH);
+    let yarn_infra = Program::new("yarn").working_dir(&ts_infra_path);
 
     log!("Deploying hyperlane ism contracts...");
     yarn_infra.clone().cmd("deploy-ism").run().join();

rust/main/utils/run-locally/src/invariants.rs → rust/main/utils/run-locally/src/invariants/mod.rs

@@ -1,4 +1,4 @@
-pub use common::SOL_MESSAGES_EXPECTED;
+pub use common::*;
 pub use post_startup_invariants::post_startup_invariants;
 pub use termination_invariants::termination_invariants_met;
 

rust/main/utils/run-locally/src/invariants/termination_invariants.rs

@@ -1,15 +1,12 @@
 use std::fs::File;
-use std::path::Path;
 
 use crate::config::Config;
 use crate::metrics::agent_balance_sum;
 use crate::utils::get_matching_lines;
 use maplit::hashmap;
 use relayer::GAS_EXPENDITURE_LOG_MESSAGE;
 
-use crate::invariants::common::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP};
 use crate::logging::log;
-use crate::solana::solana_termination_invariants_met;
 use crate::{
     fetch_metric, AGENT_LOGGING_DIR, RELAYER_METRICS_PORT, SCRAPER_METRICS_PORT,
     ZERO_MERKLE_INSERTION_KATHY_MESSAGES,
@@ -21,34 +18,20 @@ use crate::{
 pub fn termination_invariants_met(
     config: &Config,
     starting_relayer_balance: f64,
-    solana_cli_tools_path: Option<&Path>,
-    solana_config_path: Option<&Path>,
 ) -> eyre::Result<bool> {
     let eth_messages_expected = (config.kathy_messages / 2) as u32 * 2;
-    let sol_messages_expected = if config.sealevel_enabled {
-        SOL_MESSAGES_EXPECTED
-    } else {
-        0
-    };
-    let sol_messages_with_non_matching_igp = if config.sealevel_enabled {
-        SOL_MESSAGES_WITH_NON_MATCHING_IGP
-    } else {
-        0
-    };
 
     // this is total messages expected to be delivered
-    let total_messages_expected = eth_messages_expected + sol_messages_expected;
-    let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp;
+    let total_messages_expected = eth_messages_expected;
+    let total_messages_dispatched = total_messages_expected;
 
     let lengths = fetch_metric(
         RELAYER_METRICS_PORT,
         "hyperlane_submitter_queue_length",
         &hashmap! {},
     )?;
     assert!(!lengths.is_empty(), "Could not find queue length metric");
-    if lengths.iter().sum::<u32>()
-        != ZERO_MERKLE_INSERTION_KATHY_MESSAGES + sol_messages_with_non_matching_igp
-    {
+    if lengths.iter().sum::<u32>() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES {
         log!(
             "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}",
             lengths
@@ -162,20 +145,9 @@ pub fn termination_invariants_met(
         merkle_tree_max_sequence.iter().filter(|&x| *x > 0).count() as u32;
     assert_eq!(
         merkle_tree_max_sequence.iter().sum::<u32>() + non_zero_sequence_count,
-        total_messages_expected
-            + sol_messages_with_non_matching_igp
-            + (config.kathy_messages as u32 / 4) * 2
+        total_messages_expected + (config.kathy_messages as u32 / 4) * 2
     );
 
-    if let Some((solana_cli_tools_path, solana_config_path)) =
-        solana_cli_tools_path.zip(solana_config_path)
-    {
-        if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) {
-            log!("Solana termination invariants not met");
-            return Ok(false);
-        }
-    }
-
     let dispatched_messages_scraped = fetch_metric(
         SCRAPER_METRICS_PORT,
         "hyperlane_contract_sync_stored_events",
@@ -221,12 +193,13 @@ pub fn termination_invariants_met(
         log!(
             "Scraper has scraped {} delivered messages, expected {}",
             delivered_messages_scraped,
-            total_messages_expected + sol_messages_with_non_matching_igp
+            total_messages_expected
         );
         return Ok(false);
     }
 
-    let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap();
+    let ending_relayer_balance: f64 =
+        agent_balance_sum(9092).expect("Failed to get relayer agent balance");
     // Make sure the balance was correctly updated in the metrics.
     if starting_relayer_balance <= ending_relayer_balance {
         log!(