[tmpnet] Ensure GetNodeURIs returns locally-accessible URIs to ensure kube compatibility (#3973)

Author: maru-ava

tests/antithesis/config.go

@@ -102,8 +102,9 @@ func configForNewNetwork(
 	c := &Config{
 		Duration: duration,
 	}
-	c.URIs = make(CSV, len(testEnv.URIs))
-	for i, nodeURI := range testEnv.URIs {
+	localURIs := testEnv.GetNodeURIs()
+	c.URIs = make(CSV, len(localURIs))
+	for i, nodeURI := range localURIs {
 		c.URIs[i] = nodeURI.URI
 	}
 	network := testEnv.GetNetwork()

tests/e2e/p/validator_sets.go

@@ -81,8 +81,9 @@ var _ = e2e.DescribePChain("[Validator Sets]", func() {
 		require.NoError(err)
 
 		tc.By("checking that validator sets are equal across all heights for all nodes", func() {
-			pvmClients := make([]platformvm.Client, len(env.URIs))
-			for i, nodeURI := range env.URIs {
+			localURIs := env.GetNodeURIs()
+			pvmClients := make([]platformvm.Client, len(localURIs))
+			for i, nodeURI := range localURIs {
 				pvmClients[i] = platformvm.NewClient(nodeURI.URI)
 				// Ensure that the height of the target node is at least the expected height
 				tc.Eventually(

tests/e2e/x/transfer/virtuous.go

@@ -47,9 +47,12 @@ var _ = e2e.DescribeXChainSerial("[Virtuous Transfer Tx AVAX]", func() {
 
 	ginkgo.It("can issue a virtuous transfer tx for AVAX asset",
 		func() {
-			env := e2e.GetEnv(tc)
-			rpcEps := make([]string, len(env.URIs))
-			for i, nodeURI := range env.URIs {
+			var (
+				env       = e2e.GetEnv(tc)
+				localURIs = env.GetNodeURIs()
+				rpcEps    = make([]string, len(localURIs))
+			)
+			for i, nodeURI := range localURIs {
 				rpcEps[i] = nodeURI.URI
 			}
 

tests/fixture/e2e/env.go

@@ -46,8 +46,6 @@ type TestEnvironment struct {
 	RootNetworkDir string
 	// The directory where the test network configuration is stored
 	NetworkDir string
-	// URIs used to access the API endpoints of nodes of the network
-	URIs []tmpnet.NodeURI
 	// Pre-funded key for this ginkgo process
 	PreFundedKey *secp256k1.PrivateKey
 	// The duration to wait before shutting down private networks. A
@@ -66,7 +64,6 @@ func GetEnv(tc tests.TestContext) *TestEnvironment {
 	return &TestEnvironment{
 		RootNetworkDir:              env.RootNetworkDir,
 		NetworkDir:                  env.NetworkDir,
-		URIs:                        env.URIs,
 		PreFundedKey:                env.PreFundedKey,
 		PrivateNetworkShutdownDelay: env.PrivateNetworkShutdownDelay,
 		testContext:                 tc,
@@ -210,30 +207,85 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 		"not enough pre-funded keys for the requested number of parallel test processes",
 	)
 
-	uris := network.GetNodeURIs()
-	require.NotEmpty(uris, "network contains no nodes")
-	tc.Log().Info("network nodes are available",
-		zap.Any("uris", uris),
-	)
-
-	return &TestEnvironment{
+	env := &TestEnvironment{
 		RootNetworkDir:              flagVars.RootNetworkDir(),
 		NetworkDir:                  network.Dir,
-		URIs:                        uris,
 		PrivateNetworkShutdownDelay: flagVars.NetworkShutdownDelay(),
 		testContext:                 tc,
 	}
+
+	if network.DefaultRuntimeConfig.Process != nil {
+		// Display node IDs and URIs for process-based networks since the nodes are guaranteed to be network accessible
+		uris := env.GetNodeURIs()
+		require.NotEmpty(uris, "network contains no nodes")
+		tc.Log().Info("network nodes are available",
+			zap.Any("uris", uris),
+		)
+	} else {
+		// Only display node IDs for kube-based networks since the nodes may not be network accessible and
+		// port-forwarded URIs are ephemeral
+		nodeIDs := network.GetAvailableNodeIDs()
+		require.NotEmpty(nodeIDs, "network contains no nodes")
+		tc.Log().Info("network nodes are available. Not showing node URIs since kube nodes may be running remotely.",
+			zap.Strings("nodeIDs", nodeIDs),
+		)
+	}
+
+	return env
+}
+
+// Retrieve URIs for validator nodes of the shared network. The URIs
+// are only guaranteed to be accessible until the environment test
+// context is torn down (usually the duration of execution of a single
+// test).
+func (te *TestEnvironment) GetNodeURIs() []tmpnet.NodeURI {
+	var (
+		tc      = te.testContext
+		network = te.GetNetwork()
+	)
+	uris, err := network.GetNodeURIs(tc.DefaultContext(), tc.DeferCleanup)
+	require.NoError(tc, err)
+	return uris
 }
 
-// Retrieve a random URI to naively attempt to spread API load across
-// nodes.
+// Retrieve a random URI to naively attempt to spread API load across nodes.
 func (te *TestEnvironment) GetRandomNodeURI() tmpnet.NodeURI {
-	r := rand.New(rand.NewSource(time.Now().Unix())) //#nosec G404
-	nodeURI := te.URIs[r.Intn(len(te.URIs))]
-	te.testContext.Log().Info("targeting random node",
+	var (
+		tc             = te.testContext
+		r              = rand.New(rand.NewSource(time.Now().Unix())) //#nosec G404
+		network        = te.GetNetwork()
+		availableNodes = []*tmpnet.Node{}
+	)
+
+	for _, node := range network.Nodes {
+		if node.IsEphemeral {
+			// Avoid returning URIs for nodes whose lifespan is indeterminate
+			continue
+		}
+		if !node.IsRunning() {
+			// Only running nodes have URIs
+			continue
+		}
+		availableNodes = append(availableNodes, node)
+	}
+
+	require.NotEmpty(tc, availableNodes, "no available nodes to target")
+
+	// Use a local URI for the node to ensure compatibility with kube
+	randomNode := availableNodes[r.Intn(len(availableNodes))]
+	uri, cancel, err := randomNode.GetLocalURI(tc.DefaultContext())
+	require.NoError(tc, err)
+	tc.DeferCleanup(cancel)
+
+	nodeURI := tmpnet.NodeURI{
+		NodeID: randomNode.NodeID,
+		URI:    uri,
+	}
+	tc.Log().Info("targeting random node",
 		zap.Stringer("nodeID", nodeURI.NodeID),
 		zap.String("uri", nodeURI.URI),
 	)
+
 	return nodeURI
 }
 

tests/fixture/tmpnet/network.go

@@ -767,8 +767,21 @@ func (n *Network) GetNode(nodeID ids.NodeID) (*Node, error) {
 	return nil, fmt.Errorf("%s is not known to the network", nodeID)
 }
 
-func (n *Network) GetNodeURIs() []NodeURI {
-	return GetNodeURIs(n.Nodes)
+// GetNodeURIs returns the URIs of nodes in the network that are running and not ephemeral. The URIs
+// returned are guaranteed be reachable by the caller until the cleanup function is called regardless
+// of whether the nodes are running as local processes or in a kube cluster.
+func (n *Network) GetNodeURIs(ctx context.Context, deferCleanupFunc func(func())) ([]NodeURI, error) {
+	return GetNodeURIs(ctx, n.Nodes, deferCleanupFunc)
+}
+
+// GetAvailableNodeIDs returns the node IDs of nodes in the network that are running and not ephemeral.
+func (n *Network) GetAvailableNodeIDs() []string {
+	availableNodes := FilterAvailableNodes(n.Nodes)
+	ids := make([]string, len(availableNodes))
+	for _, node := range availableNodes {
+		ids = append(ids, node.NodeID.String())
+	}
+	return ids
 }
 
 // Retrieves bootstrap IPs and IDs for all non-ephemeral nodes except the skipped one

tests/fixture/tmpnet/utils.go

@@ -53,23 +53,42 @@ type NodeURI struct {
 	URI    string
 }
 
-func GetNodeURIs(nodes []*Node) []NodeURI {
-	uris := make([]NodeURI, 0, len(nodes))
+// GetNodeURIs returns the URIs of the provided nodes that are running and not ephemeral. The URIs returned
+// are guaranteed be reachable by the caller until the cleanup function is called regardless of whether the
+// nodes are running as local processes or in a kube cluster.
+func GetNodeURIs(ctx context.Context, nodes []*Node, deferCleanupFunc func(func())) ([]NodeURI, error) {
+	availableNodes := FilterAvailableNodes(nodes)
+	uris := make([]NodeURI, 0, len(availableNodes))
+	for _, node := range availableNodes {
+		uri, cancel, err := node.GetLocalURI(ctx)
+		if err != nil {
+			return nil, err
+		}
+		deferCleanupFunc(cancel)
+		uris = append(uris, NodeURI{
+			NodeID: node.NodeID,
+			URI:    uri,
+		})
+	}
+
+	return uris, nil
+}
+
+// FilteredAvailableNodes filters the provided nodes by whether they are running and not ephemeral.
+func FilterAvailableNodes(nodes []*Node) []*Node {
+	filteredNodes := make([]*Node, 0, len(nodes))
 	for _, node := range nodes {
 		if node.IsEphemeral {
 			// Avoid returning URIs for nodes whose lifespan is indeterminate
 			continue
 		}
-		// Only append URIs that are not empty. A node may have an
-		// empty URI if it is not currently running.
-		if node.IsRunning() {
-			uris = append(uris, NodeURI{
-				NodeID: node.NodeID,
-				URI:    node.URI,
-			})
+		if !node.IsRunning() {
+			// Only running nodes have URIs
+			continue
 		}
+		filteredNodes = append(filteredNodes, node)
 	}
-	return uris
+	return filteredNodes
 }
 
 // Marshal to json with default prefix and indent.