diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index 157306b3dd1..3333bc49358 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -197,25 +197,43 @@ runner-test-matrix: E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos CI: "true" - CTF_CONFIGS: "environment-ci.toml" + CTF_CONFIGS: "environment-one-don-ci.toml" # Anvil developer key, not a secret PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - - id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_TwoDons + - id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_GatewayDon path: system-tests/tests/smoke/capabilities/workflow_test.go test_env_type: docker runs_on: ubuntu-latest triggers: - PR Workflow Engine E2E Core Tests - Nightly E2E Tests - test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_TwoDons_LivePrice)$" -timeout 30m -count=1 -test.parallel=1 -json + test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_GatewayDon_MockedPrice)$" -timeout 30m -count=1 -test.parallel=1 -json pyroscope_env: ci-smoke-capabilities-evm-simulated test_env_vars: E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos CI: "true" - CTF_CONFIGS: "environment-multi-don-ci.toml" + CTF_CONFIGS: "environment-two-dons-ci.toml" + # Anvil developer key, not a secret + PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + + - id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_ThreeDons + path: system-tests/tests/smoke/capabilities/workflow_test.go + test_env_type: docker + runs_on: ubuntu-latest + triggers: + - PR Workflow Engine E2E Core Tests + - Nightly E2E Tests + test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_ThreeDons_LivePrice)$" -timeout 30m -count=1 -test.parallel=1 -json + pyroscope_env: ci-smoke-capabilities-evm-simulated + test_env_vars: + E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins + E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version + GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos + CI: "true" + CTF_CONFIGS: "environment-three-dons-ci.toml" # Anvil developer key, not a secret PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/deployment/environment/devenv/don.go b/deployment/environment/devenv/don.go index df360dc0845..55353c93f51 100644 --- a/deployment/environment/devenv/don.go +++ b/deployment/environment/devenv/don.go @@ -263,6 +263,7 @@ func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChai return fmt.Errorf("no OCR2 key bundle id found for node %s", n.Name) } n.Ocr2KeyBundleID = ocr2BundleId + // fetch node labels to know if the node is bootstrap or plugin // if multi address is set, then it's a bootstrap node isBootstrap := n.multiAddr != "" diff --git a/system-tests/lib/cre/contracts/contracts.go b/system-tests/lib/cre/contracts/contracts.go index c86eaf79034..a18a2124add 100644 --- a/system-tests/lib/cre/contracts/contracts.go +++ b/system-tests/lib/cre/contracts/contracts.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" keystonenode "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" ) @@ -30,13 +31,19 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { return errors.Wrap(err, "input validation failed") } - donCapabilities := make([]keystone_changeset.DonCapabilities, 0, len(input.DonTopology.MetaDons)) + donCapabilities := make([]keystone_changeset.DonCapabilities, 0, len(input.Topology.DonsMetadata)) + + for _, donMetadata := range input.Topology.DonsMetadata { + // if it's only a gateway DON, we don't want to register it with the Capabilities Registry + // since it doesn't have any capabilities + if flags.HasOnlyOneFlag(donMetadata.Flags, types.GatewayDON) { + continue + } - for _, metaDon := range input.DonTopology.MetaDons { var capabilities []keystone_changeset.DONCapabilityWithConfig // check what capabilities each DON has and register them with Capabilities Registry contract - if flags.HasFlag(metaDon.Flags, types.CronCapability) { + if flags.HasFlag(donMetadata.Flags, types.CronCapability) { capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{ Capability: kcr.CapabilitiesRegistryCapability{ LabelledName: "cron-trigger", @@ -47,7 +54,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { }) } - if flags.HasFlag(metaDon.Flags, types.CustomComputeCapability) { + if flags.HasFlag(donMetadata.Flags, types.CustomComputeCapability) { capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{ Capability: kcr.CapabilitiesRegistryCapability{ LabelledName: "custom-compute", @@ -58,7 +65,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { }) } - if flags.HasFlag(metaDon.Flags, types.OCR3Capability) { + if flags.HasFlag(donMetadata.Flags, types.OCR3Capability) { capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{ Capability: kcr.CapabilitiesRegistryCapability{ LabelledName: "offchain_reporting", @@ -70,7 +77,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { }) } - if flags.HasFlag(metaDon.Flags, types.WriteEVMCapability) { + if flags.HasFlag(donMetadata.Flags, types.WriteEVMCapability) { capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{ Capability: kcr.CapabilitiesRegistryCapability{ LabelledName: "write_geth-testnet", @@ -84,28 +91,33 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { // Add support for new capabilities here as needed - donPeerIDs := make([]string, len(metaDon.DON.Nodes)-1) - for i, node := range metaDon.DON.Nodes { - if i == 0 { - continue - } + workerNodes, workerNodesErr := node.FindManyWithLabel(donMetadata.NodesMetadata, &types.Label{ + Key: node.NodeTypeKey, + Value: types.WorkerNode, + }, node.EqualLabels) + if workerNodesErr != nil { + return errors.Wrap(workerNodesErr, "failed to find worker nodes") + } + + donPeerIDs := make([]string, len(workerNodes)) + for i, node := range workerNodes { p2pID, err := keystonenode.ToP2PID(node, keystonenode.NoOpTransformFn) if err != nil { - return errors.Wrapf(err, "failed to get p2p id for node %s", node.Name) + return errors.Wrapf(err, "failed to get p2p id for node %d", i) } - donPeerIDs[i-1] = p2pID + donPeerIDs[i] = p2pID } // we only need to assign P2P IDs to NOPs, since `ConfigureInitialContractsChangeset` method // will take care of creating DON to Nodes mapping nop := keystone_changeset.NOP{ - Name: fmt.Sprintf("NOP for %s DON", metaDon.NodeOutput.NodeSetName), + Name: fmt.Sprintf("NOP for %s DON", donMetadata.Name), Nodes: donPeerIDs, } - donName := metaDon.NodeOutput.NodeSetName + "-don" + donName := donMetadata.Name + "-don" donCapabilities = append(donCapabilities, keystone_changeset.DonCapabilities{ Name: donName, F: 1, @@ -116,10 +128,19 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error { var transmissionSchedule []int - for _, metaDon := range input.DonTopology.MetaDons { + for _, metaDon := range input.Topology.DonsMetadata { if flags.HasFlag(metaDon.Flags, types.OCR3Capability) { + workerNodes, workerNodesErr := node.FindManyWithLabel(metaDon.NodesMetadata, &types.Label{ + Key: node.NodeTypeKey, + Value: types.WorkerNode, + }, node.EqualLabels) + + if workerNodesErr != nil { + return errors.Wrap(workerNodesErr, "failed to find worker nodes") + } + // this schedule makes sure that all worker nodes are transmitting OCR3 reports - transmissionSchedule = []int{len(metaDon.DON.Nodes) - 1} + transmissionSchedule = []int{len(workerNodes)} break } } diff --git a/system-tests/lib/cre/debug/debug.go b/system-tests/lib/cre/debug/debug.go index 32697a92bd5..cfbe520d30a 100644 --- a/system-tests/lib/cre/debug/debug.go +++ b/system-tests/lib/cre/debug/debug.go @@ -14,7 +14,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/rs/zerolog" - ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" "github.com/smartcontractkit/chainlink-testing-framework/seth" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" @@ -40,8 +39,8 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) { } }() - for _, donTopology := range input.DonTopology.MetaDons { - logFiles, err := getLogFileHandles(testName, l, donTopology.NodeOutput.Output) + for _, debugDon := range input.DebugDons { + logFiles, err := getLogFileHandles(testName, l, debugDon) if err != nil { l.Error().Err(err).Msg("Failed to get log file handles. No debug information will be printed") return @@ -50,9 +49,9 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) { allLogFiles = append(allLogFiles, logFiles...) // assuming one bootstrap node - workflowNodeCount := len(donTopology.NodeOutput.CLNodes) - 1 + workflowNodeCount := len(debugDon.ContainerNames) - 1 - if flags.HasFlag(donTopology.Flags, types.WorkflowDON) { + if flags.HasFlag(debugDon.Flags, types.WorkflowDON) { if !checkIfWorkflowWasExecuting(logFiles, workflowNodeCount) { l.Error().Msg("❌ Workflow was not executing") return @@ -60,7 +59,7 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) { l.Info().Msg("✅ Workflow was executing") } - if flags.HasFlag(donTopology.Flags, types.OCR3Capability) { + if flags.HasFlag(debugDon.Flags, types.OCR3Capability) { if !checkIfOCRWasExecuting(logFiles, workflowNodeCount) { l.Error().Msg("❌ OCR was not executing") return @@ -68,7 +67,7 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) { l.Info().Msg("✅ OCR was executing") } - if flags.HasFlag(donTopology.Flags, types.WriteEVMCapability) { + if flags.HasFlag(debugDon.Flags, types.WriteEVMCapability) { if !checkIfAtLeastOneReportWasSent(logFiles, workflowNodeCount) { l.Error().Msg("❌ Reports were not sent") return @@ -167,21 +166,18 @@ func ReportTransmissions(logFiles []*os.File, l zerolog.Logger, wsRPCURL string) } } -func getLogFileHandles(testName string, l zerolog.Logger, ns *ns.Output) ([]*os.File, error) { +func getLogFileHandles(testName string, l zerolog.Logger, debugDon *types.DebugDon) ([]*os.File, error) { var logFiles []*os.File var belongsToCurrentEnv = func(filePath string) bool { - for i, clNode := range ns.CLNodes { - if clNode == nil { - continue - } - + for i, containerName := range debugDon.ContainerNames { + // TODO check corresponding flag when looking for bootstrap node // skip the first node, as it's the bootstrap node if i == 0 { continue } - if strings.EqualFold(filePath, clNode.Node.ContainerName+".log") { + if strings.EqualFold(filePath, containerName+".log") { return true } } @@ -204,7 +200,7 @@ func getLogFileHandles(testName string, l zerolog.Logger, ns *ns.Output) ([]*os. return nil }) - expectedLogCount := len(ns.CLNodes) - 1 + expectedLogCount := len(debugDon.ContainerNames) - 1 if len(logFiles) != expectedLogCount { l.Warn().Int("Expected", expectedLogCount).Int("Got", len(logFiles)).Msg("Number of log files does not match number of worker nodes. Some logs might be missing.") } diff --git a/system-tests/lib/cre/don/config/definitions.go b/system-tests/lib/cre/don/config/definitions.go index 770a8594364..4eead38b91f 100644 --- a/system-tests/lib/cre/don/config/definitions.go +++ b/system-tests/lib/cre/don/config/definitions.go @@ -49,7 +49,7 @@ func BootstrapEVM(donBootstrapNodePeerID string, chainID uint64, capabilitiesReg ) } -func BoostrapDon2DonPeering(peeringData types.PeeringData) string { +func BoostrapDon2DonPeering(peeringData types.CapabilitiesPeeringData) string { return fmt.Sprintf(` [Capabilities.Peering.V2] Enabled = true @@ -71,7 +71,7 @@ func BoostrapDon2DonPeering(peeringData types.PeeringData) string { // // so that we are future-proof (for bootstrap too!) // we'd need to have capabilitiesRegistryChainID too -func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData types.PeeringData, chainID uint64, capabilitiesRegistryAddress common.Address, httpRPC, wsRPC string) string { +func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData types.CapabilitiesPeeringData, chainID uint64, capabilitiesRegistryAddress common.Address, httpRPC, wsRPC string) string { return fmt.Sprintf(` [Feature] LogPoller = true @@ -93,6 +93,7 @@ func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData [[EVM]] ChainID = '%d' + AutoCreateKey = false [[EVM.Nodes]] Name = 'anvil' diff --git a/system-tests/lib/cre/don/config/por/por.go b/system-tests/lib/cre/don/config/por/por.go index afd67a8672d..2556e1b9ab7 100644 --- a/system-tests/lib/cre/don/config/por/por.go +++ b/system-tests/lib/cre/don/config/por/por.go @@ -6,21 +6,23 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" - "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" - libc "github.com/smartcontractkit/chainlink/system-tests/lib/conversions" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/config" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" keystoneflags "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" - "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" + cretypes "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) -func GenerateConfigs(input types.GeneratePoRConfigsInput) (types.NodeIndexToConfigOverrides, error) { +func GenerateConfigs(input cretypes.GeneratePoRConfigsInput) (cretypes.NodeIndexToConfigOverride, error) { if err := input.Validate(); err != nil { return nil, errors.Wrap(err, "input validation failed") } - configOverrides := make(types.NodeIndexToConfigOverrides) + configOverrides := make(cretypes.NodeIndexToConfigOverride) + + // if it's only a gateway DON, we don't need to generate any extra configuration, the default one will do + if keystoneflags.HasFlag(input.Flags, cretypes.GatewayDON) && (!keystoneflags.HasFlag(input.Flags, cretypes.WorkflowDON) && !keystoneflags.HasFlag(input.Flags, cretypes.CapabilitiesDON)) { + return configOverrides, nil + } chainIDInt, err := strconv.Atoi(input.BlockchainOutput.ChainID) if err != nil { @@ -28,21 +30,23 @@ func GenerateConfigs(input types.GeneratePoRConfigsInput) (types.NodeIndexToConf } chainIDUint64 := libc.MustSafeUint64(int64(chainIDInt)) - // find bootstrap node - bootstrapNode, err := node.FindOneWithLabel(input.Don, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.BootstrapNode)}) + // find bootstrap node for the Don + var donBootstrapNodeHost string + var donBootstrapNodePeerID string + + bootstrapNode, err := node.FindOneWithLabel(input.DonMetadata.NodesMetadata, &cretypes.Label{Key: node.NodeTypeKey, Value: cretypes.BootstrapNode}, node.EqualLabels) if err != nil { return nil, errors.Wrap(err, "failed to find bootstrap node") } - donBootstrapNodePeerID, err := node.ToP2PID(*bootstrapNode, node.KeyExtractingTransformFn) + donBootstrapNodePeerID, err = node.ToP2PID(bootstrapNode, node.KeyExtractingTransformFn) if err != nil { return nil, errors.Wrap(err, "failed to get bootstrap node peer ID") } - var donBootstrapNodeHost string - for _, label := range bootstrapNode.Labels() { + for _, label := range bootstrapNode.Labels { if label.Key == node.HostLabelKey { - donBootstrapNodeHost = *label.Value + donBootstrapNodeHost = label.Value break } } @@ -52,48 +56,54 @@ func GenerateConfigs(input types.GeneratePoRConfigsInput) (types.NodeIndexToConf } var nodeIndex int - for _, label := range bootstrapNode.Labels() { - if label.Key == node.NodeIndexKey { - nodeIndex, err = strconv.Atoi(*label.Value) + for _, label := range bootstrapNode.Labels { + if label.Key == node.IndexKey { + nodeIndex, err = strconv.Atoi(label.Value) if err != nil { return nil, errors.Wrap(err, "failed to convert node index to int") } + break } } - // generat configuration for the bootstrap node + // generate configuration for the bootstrap node configOverrides[nodeIndex] = config.BootstrapEVM(donBootstrapNodePeerID, chainIDUint64, input.CapabilitiesRegistryAddress, input.BlockchainOutput.Nodes[0].DockerInternalHTTPUrl, input.BlockchainOutput.Nodes[0].DockerInternalWSUrl) - if keystoneflags.HasFlag(input.Flags, types.WorkflowDON) { + if keystoneflags.HasFlag(input.Flags, cretypes.WorkflowDON) { configOverrides[nodeIndex] += config.BoostrapDon2DonPeering(input.PeeringData) - - if input.GatewayConnectorOutput == nil { - return nil, errors.New("GatewayConnectorOutput is required for Workflow DON") - } - input.GatewayConnectorOutput.Host = donBootstrapNodeHost } // find worker nodes - workflowNodeSet, err := node.FindManyWithLabel(input.Don, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.WorkerNode)}) + workflowNodeSet, err := node.FindManyWithLabel(input.DonMetadata.NodesMetadata, &cretypes.Label{Key: node.NodeTypeKey, Value: cretypes.WorkerNode}, node.EqualLabels) if err != nil { return nil, errors.Wrap(err, "failed to find worker nodes") } for i := range workflowNodeSet { var nodeIndex int - for _, label := range workflowNodeSet[i].Labels() { - if label.Key == node.NodeIndexKey { - nodeIndex, err = strconv.Atoi(*label.Value) + for _, label := range workflowNodeSet[i].Labels { + if label.Key == node.IndexKey { + nodeIndex, err = strconv.Atoi(label.Value) if err != nil { return nil, errors.Wrap(err, "failed to convert node index to int") } } } + // for now we just assume that every worker node is connected to one EVM chain configOverrides[nodeIndex] = config.WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost, input.PeeringData, chainIDUint64, input.CapabilitiesRegistryAddress, input.BlockchainOutput.Nodes[0].DockerInternalHTTPUrl, input.BlockchainOutput.Nodes[0].DockerInternalWSUrl) - nodeEthAddr := common.HexToAddress(workflowNodeSet[i].AccountAddr[chainIDUint64]) + var nodeEthAddr common.Address + for _, label := range workflowNodeSet[i].Labels { + if label.Key == node.EthAddressKey { + if label.Value == "" { + return nil, errors.New("eth address label value is empty") + } + nodeEthAddr = common.HexToAddress(label.Value) + break + } + } - if keystoneflags.HasFlag(input.Flags, types.WriteEVMCapability) { + if keystoneflags.HasFlag(input.Flags, cretypes.WriteEVMCapability) { configOverrides[nodeIndex] += config.WorkerWriteEMV( nodeEthAddr, input.ForwarderAddress, @@ -101,14 +111,14 @@ func GenerateConfigs(input types.GeneratePoRConfigsInput) (types.NodeIndexToConf } // if it's workflow DON configure workflow registry - if keystoneflags.HasFlag(input.Flags, types.WorkflowDON) { + if keystoneflags.HasFlag(input.Flags, cretypes.WorkflowDON) { configOverrides[nodeIndex] += config.WorkerWorkflowRegistry( input.WorkflowRegistryAddress, chainIDUint64) } // workflow DON nodes always needs gateway connector, otherwise they won't be able to fetch the workflow // it's also required by custom compute, which can only run on workflow DON nodes - if keystoneflags.HasFlag(input.Flags, types.WorkflowDON) || keystoneflags.HasFlag(input.Flags, types.CustomComputeCapability) { + if keystoneflags.HasFlag(input.Flags, cretypes.WorkflowDON) || keystoneflags.HasFlag(input.Flags, cretypes.CustomComputeCapability) { configOverrides[nodeIndex] += config.WorkerGateway( nodeEthAddr, chainIDUint64, diff --git a/system-tests/lib/cre/don/don.go b/system-tests/lib/cre/don/don.go index 31687809dc0..714337ae544 100644 --- a/system-tests/lib/cre/don/don.go +++ b/system-tests/lib/cre/don/don.go @@ -1,121 +1,250 @@ package don import ( + "fmt" "regexp" - "testing" + "strconv" "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/smartcontractkit/chainlink-testing-framework/framework" - ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" - "github.com/smartcontractkit/chainlink/deployment/environment/devenv" libc "github.com/smartcontractkit/chainlink/system-tests/lib/conversions" - "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/config" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" - "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" + cretypes "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" + "github.com/smartcontractkit/chainlink/system-tests/lib/crypto" ) -func Configure(t *testing.T, testLogger zerolog.Logger, input types.ConfigureDonInput) (*types.ConfigureDonOutput, error) { +func CreateJobs(testLogger zerolog.Logger, input cretypes.CreateJobsInput) error { if err := input.Validate(); err != nil { - return nil, errors.Wrap(err, "input validation failed") + return errors.Wrap(err, "input validation failed") } - for i, donTopology := range input.DonTopology.MetaDons { - if configOverrides, ok := input.DonToConfigOverrides[donTopology.ID]; ok { - for j, configOverride := range configOverrides { - if len(donTopology.NodeInput.NodeSpecs)-1 < j { - return nil, errors.Errorf("config override index out of bounds: %d", j) - } - donTopology.NodeInput.NodeSpecs[j].Node.TestConfigOverrides = configOverride - } - var setErr error - input.DonTopology.MetaDons[i].NodeOutput, setErr = config.Set(t, donTopology.NodeInput, input.BlockchainOutput) - if setErr != nil { - return nil, errors.Wrap(setErr, "failed to set node output") - } - } - } - - nodeOutputs := make([]*types.WrappedNodeOutput, 0, len(input.DonTopology.MetaDons)) - for i := range input.DonTopology.MetaDons { - nodeOutputs = append(nodeOutputs, input.DonTopology.MetaDons[i].NodeOutput) - } - - // after restarting the nodes, we need to reinitialize the JD clients otherwise - // communication between JD and nodes will fail due to invalidated session cookie - // TODO remove if our idea with pre-generating & importing keys works and we do not need to restart the nodes - jdOutput, jdErr := jobs.ReinitialiseJDClients(input.CldEnv, input.JdOutput, nodeOutputs...) - if jdErr != nil { - return nil, errors.Wrap(jdErr, "failed to reinitialize JD clients") - } - for _, donTopology := range input.DonTopology.MetaDons { - if jobSpecs, ok := input.DonToJobSpecs[donTopology.ID]; ok { - createErr := jobs.Create(input.CldEnv.Offchain, donTopology.DON, donTopology.Flags, jobSpecs) + for _, don := range input.DonTopology.DonsWithMetadata { + if jobSpecs, ok := input.DonToJobSpecs[don.ID]; ok { + createErr := jobs.Create(input.CldEnv.Offchain, don.DON, don.Flags, jobSpecs) if createErr != nil { - return nil, errors.Wrapf(createErr, "failed to create jobs for DON %d", donTopology.ID) + return errors.Wrapf(createErr, "failed to create jobs for DON %d", don.ID) } } else { - testLogger.Warn().Msgf("No job specs found for DON %d", donTopology.ID) + testLogger.Warn().Msgf("No job specs found for DON %d", don.ID) } } - return &types.ConfigureDonOutput{ - JdOutput: &jdOutput.Offchain, - }, nil + return nil } -func BuildDONTopology(dons []*devenv.DON, nodeSetInput []*types.CapabilitiesAwareNodeSet, nodeSetOutput []*types.WrappedNodeOutput) (*types.DonTopology, error) { - donWithMeta := make([]*types.DonWithMetadata, len(dons)) +func BuildTopology(nodeSetInput []*cretypes.CapabilitiesAwareNodeSet) (*cretypes.Topology, error) { + topology := &cretypes.Topology{} + donsWithMetadata := make([]*cretypes.DonMetadata, len(nodeSetInput)) // one DON to do everything - if len(dons) == 1 { + if len(nodeSetInput) == 1 { flags, err := flags.NodeSetFlags(nodeSetInput[0]) if err != nil { - return nil, errors.Wrapf(err, "failed to convert string flags to bitmap for nodeset %s", nodeSetInput[0].Name) + return nil, errors.Wrapf(err, "failed to get flags for nodeset %s", nodeSetInput[0].Name) } - donWithMeta[0] = &types.DonWithMetadata{ - DON: dons[0], - NodeInput: nodeSetInput[0], - NodeOutput: nodeSetOutput[0], - ID: 1, - Flags: flags, + donsWithMetadata[0] = &cretypes.DonMetadata{ + ID: 1, + Flags: flags, + NodesMetadata: make([]*cretypes.NodeMetadata, len(nodeSetInput[0].NodeSpecs)), + Name: nodeSetInput[0].Name, } } else { - for i := range dons { + for i := range nodeSetInput { flags, err := flags.NodeSetFlags(nodeSetInput[i]) if err != nil { - return nil, errors.Wrapf(err, "failed to convert string flags to bitmap for nodeset %s", nodeSetInput[i].Name) + return nil, errors.Wrapf(err, "failed to get flags for nodeset %s", nodeSetInput[i].Name) + } + + donsWithMetadata[i] = &cretypes.DonMetadata{ + ID: libc.MustSafeUint32(i + 1), + Flags: flags, + NodesMetadata: make([]*cretypes.NodeMetadata, len(nodeSetInput[i].NodeSpecs)), + Name: nodeSetInput[i].Name, } + } + } - donWithMeta[i] = &types.DonWithMetadata{ - DON: dons[i], - NodeInput: nodeSetInput[i], - NodeOutput: nodeSetOutput[i], - ID: libc.MustSafeUint32(i + 1), - Flags: flags, + for i, donMetadata := range donsWithMetadata { + for j := range donMetadata.NodesMetadata { + nodeWithLabels := cretypes.NodeMetadata{} + nodeType := cretypes.WorkerNode + if nodeSetInput[i].BootstrapNodeIndex != -1 && j == nodeSetInput[i].BootstrapNodeIndex { + nodeType = cretypes.BootstrapNode } + nodeWithLabels.Labels = append(nodeWithLabels.Labels, &cretypes.Label{ + Key: node.NodeTypeKey, + Value: nodeType, + }) + + // TODO this will only work with Docker, for CRIB we need a different approach + // that will need to be aware of namespace name and node naming pattern + host := fmt.Sprintf("%s-node%d", donMetadata.Name, j) + + if nodeSetInput[i].GatewayNodeIndex != -1 && j == nodeSetInput[i].GatewayNodeIndex { + nodeWithLabels.Labels = append(nodeWithLabels.Labels, &cretypes.Label{ + Key: node.ExtraRolesKey, + Value: cretypes.GatewayNode, + }) + + topology.GatewayConnectorOutput = &cretypes.GatewayConnectorOutput{ + Path: "/node", + Port: 5003, + Host: host, + // do not set gateway connector dons, they will be resolved automatically + } + } + + nodeWithLabels.Labels = append(nodeWithLabels.Labels, &cretypes.Label{ + Key: node.IndexKey, + Value: strconv.Itoa(j), + }) + + nodeWithLabels.Labels = append(nodeWithLabels.Labels, &cretypes.Label{ + Key: node.HostLabelKey, + Value: host, + }) + + donsWithMetadata[i].NodesMetadata[j] = &nodeWithLabels } } - maybeID, err := flags.OneDONTopologyWithFlag(donWithMeta, types.WorkflowDON) + maybeID, err := flags.OneDonMetadataWithFlag(donsWithMetadata, cretypes.WorkflowDON) if err != nil { return nil, errors.Wrap(err, "failed to get workflow DON ID") } - return &types.DonTopology{ - MetaDons: donWithMeta, - WorkflowDONID: maybeID.ID, - }, nil + topology.DonsMetadata = donsWithMetadata + topology.WorkflowDONID = maybeID.ID + + return topology, nil +} + +func AddKeysToTopology(topology *cretypes.Topology, keys *cretypes.GenerateKeysOutput) (*cretypes.Topology, error) { + if topology == nil { + return nil, errors.New("topology is nil") + } + + if keys == nil { + return nil, errors.New("keys is nil") + } + + for _, donMetadata := range topology.DonsMetadata { + if p2pKeys, ok := keys.P2PKeys[donMetadata.ID]; ok { + for idx, nodeMetadata := range donMetadata.NodesMetadata { + nodeMetadata.Labels = append(nodeMetadata.Labels, &cretypes.Label{ + Key: node.NodeP2PIDKey, + Value: p2pKeys.PeerIDs[idx], + }) + } + } + + if evmKeys, ok := keys.EVMKeys[donMetadata.ID]; ok { + for idx, nodeMetadata := range donMetadata.NodesMetadata { + nodeMetadata.Labels = append(nodeMetadata.Labels, &cretypes.Label{ + Key: node.EthAddressKey, + Value: evmKeys.PublicAddresses[idx].Hex(), + }) + } + } + } + + return topology, nil +} + +func GenereteKeys(input *cretypes.GenerateKeysInput) (*cretypes.GenerateKeysOutput, error) { + if input == nil { + return nil, errors.New("input is nil") + } + + if err := input.Validate(); err != nil { + return nil, errors.Wrap(err, "input validation failed") + } + + output := &cretypes.GenerateKeysOutput{ + EVMKeys: make(cretypes.DonsToEVMKeys), + P2PKeys: make(cretypes.DonsToP2PKeys), + } + + for _, donMetadata := range input.Topology.DonsMetadata { + if input.GenerateP2PKeys { + p2pKeys, err := crypto.GenerateP2PKeys(input.Password, len(donMetadata.NodesMetadata)) + if err != nil { + return nil, errors.Wrap(err, "failed to generate P2P keys") + } + output.P2PKeys[donMetadata.ID] = p2pKeys + } + + if len(input.GenerateEVMKeysForChainIDs) > 0 { + evmKeys, err := crypto.GenerateEVMKeys(input.Password, len(donMetadata.NodesMetadata)) + if err != nil { + return nil, errors.Wrap(err, "failed to generate EVM keys") + } + evmKeys.ChainIDs = append(evmKeys.ChainIDs, input.GenerateEVMKeysForChainIDs...) + + output.EVMKeys[donMetadata.ID] = evmKeys + } + } + + return output, nil } // In order to whitelist host IP in the gateway, we need to resolve the host.docker.internal to the host IP, // and since CL image doesn't have dig or nslookup, we need to use curl. -func ResolveHostDockerInternaIP(testLogger zerolog.Logger, nsOutput *ns.Output) (string, error) { - containerName := nsOutput.CLNodes[0].Node.ContainerName +func ResolveHostDockerInternaIP(testLogger zerolog.Logger, containerName string) (string, error) { + if isCurlInstalled(containerName) { + return resolveDockerHostWithCurl(containerName) + } else if isNsLookupInstalled(containerName) { + return resolveDockerHostWithNsLookup(containerName) + } + + return "", errors.New("neither curl nor nslookup is installed") +} + +func isNsLookupInstalled(containerName string) bool { + cmd := []string{"which", "nslookup"} + output, err := framework.ExecContainer(containerName, cmd) + + if err != nil || output == "" { + return false + } + + return true +} + +func resolveDockerHostWithNsLookup(containerName string) (string, error) { + cmd := []string{"nslookup", "host.docker.internal"} + output, err := framework.ExecContainer(containerName, cmd) + if err != nil { + return "", err + } + + re := regexp.MustCompile(`host.docker.internal(\n|\r)Address:\s+([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`) + matches := re.FindStringSubmatch(output) + if len(matches) < 2 { + return "", errors.New("failed to extract IP address from curl output") + } + + return matches[2], nil +} + +func isCurlInstalled(containerName string) bool { + cmd := []string{"which", "curl"} + output, err := framework.ExecContainer(containerName, cmd) + + if err != nil || output == "" { + return false + } + + return true +} + +func resolveDockerHostWithCurl(containerName string) (string, error) { cmd := []string{"curl", "-v", "http://host.docker.internal"} output, err := framework.ExecContainer(containerName, cmd) if err != nil { @@ -125,11 +254,8 @@ func ResolveHostDockerInternaIP(testLogger zerolog.Logger, nsOutput *ns.Output) re := regexp.MustCompile(`.*Trying ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+).*`) matches := re.FindStringSubmatch(output) if len(matches) < 2 { - testLogger.Error().Msgf("failed to extract IP address from curl output:\n%s", output) return "", errors.New("failed to extract IP address from curl output") } - testLogger.Info().Msgf("Resolved host.docker.internal to %s", matches[1]) - return matches[1], nil } diff --git a/system-tests/lib/cre/don/jobs/definitions.go b/system-tests/lib/cre/don/jobs/definitions.go index c8997024664..03ed8a4c588 100644 --- a/system-tests/lib/cre/don/jobs/definitions.go +++ b/system-tests/lib/cre/don/jobs/definitions.go @@ -2,22 +2,17 @@ package jobs import ( "fmt" - "strconv" "strings" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" - "github.com/smartcontractkit/chainlink/deployment/environment/devenv" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) var ( - NoExtraAllowedPorts = []int{} - NoExtraAllowedIPs = []string{} - DefaultAllowedPorts = []int{80, 443} ) @@ -46,16 +41,36 @@ func BootstrapOCR3(nodeID string, ocr3CapabilityAddress common.Address, chainID } } -func BootstrapGateway(don *devenv.DON, chainID uint64, donID uint32, extraAllowedPorts []int, extraAllowedIps []string, gatewayConnectorData types.GatewayConnectorOutput) *jobv1.ProposeJobRequest { - var gatewayMembers string - for i := 1; i < len(don.Nodes); i++ { - gatewayMembers += fmt.Sprintf(` +func AnyGateway(bootstrapNodeID string, chainID uint64, donID uint32, extraAllowedPorts []int, extraAllowedIps []string, gatewayConnectorData types.GatewayConnectorOutput) *jobv1.ProposeJobRequest { + var gatewayDons string + + for _, don := range gatewayConnectorData.Dons { + var gatewayMembers string + + for i := 0; i < len(don.MembersEthAddresses); i++ { + gatewayMembers += fmt.Sprintf(` [[gatewayConfig.Dons.Members]] Address = "%s" Name = "Node %d"`, - don.Nodes[i].AccountAddr[chainID], - i, - ) + don.MembersEthAddresses[i], + i+1, + ) + } + + gatewayDons += fmt.Sprintf(` + [[gatewayConfig.Dons]] + DonId = "%d" + F = 1 + HandlerName = "web-api-capabilities" + [gatewayConfig.Dons.HandlerConfig] + MaxAllowedMessageAgeSec = 1_000 + [gatewayConfig.Dons.HandlerConfig.NodeRateLimiter] + GlobalBurst = 10 + GlobalRPS = 50 + PerSenderBurst = 10 + PerSenderRPS = 10 + %s + `, don.ID, gatewayMembers) } uuid := uuid.NewString() @@ -71,18 +86,7 @@ func BootstrapGateway(don *devenv.DON, chainID uint64, donID uint32, extraAllowe AuthGatewayId = "por_gateway" AuthTimestampToleranceSec = 5 HeartbeatIntervalSec = 20 - [[gatewayConfig.Dons]] - DonId = "%s" - F = 1 - HandlerName = "web-api-capabilities" - [gatewayConfig.Dons.HandlerConfig] - MaxAllowedMessageAgeSec = 1_000 - [gatewayConfig.Dons.HandlerConfig.NodeRateLimiter] - GlobalBurst = 10 - GlobalRPS = 50 - PerSenderBurst = 10 - PerSenderRPS = 10 - %s + %s [gatewayConfig.NodeServerConfig] HandshakeTimeoutMillis = 1_000 MaxRequestBytes = 100_000 @@ -106,8 +110,7 @@ func BootstrapGateway(don *devenv.DON, chainID uint64, donID uint32, extraAllowe `, uuid, uuid[0:8], - strconv.FormatUint(uint64(donID), 10), - gatewayMembers, + gatewayDons, gatewayConnectorData.Path, gatewayConnectorData.Port, ) @@ -140,7 +143,7 @@ func BootstrapGateway(don *devenv.DON, chainID uint64, donID uint32, extraAllowe } return &jobv1.ProposeJobRequest{ - NodeId: don.Nodes[0].NodeID, + NodeId: bootstrapNodeID, Spec: gatewayJobSpec, } } @@ -174,7 +177,7 @@ func WorkerStandardCapability(nodeID, name, command, config string) *jobv1.Propo } } -func WorkerOCR3(nodeID string, ocr3CapabilityAddress, nodeEthAddress common.Address, ocr2KeyBundleID string, ocrPeeringData types.OCRPeeringData, chainID uint64) *jobv1.ProposeJobRequest { +func WorkerOCR3(nodeID string, ocr3CapabilityAddress common.Address, nodeEthAddress, ocr2KeyBundleID string, ocrPeeringData types.OCRPeeringData, chainID uint64) *jobv1.ProposeJobRequest { uuid := uuid.NewString() return &jobv1.ProposeJobRequest{ diff --git a/system-tests/lib/cre/don/jobs/jobs.go b/system-tests/lib/cre/don/jobs/jobs.go index 132d0e146f9..46dc9b3865b 100644 --- a/system-tests/lib/cre/don/jobs/jobs.go +++ b/system-tests/lib/cre/don/jobs/jobs.go @@ -22,11 +22,39 @@ var SupportedJobs = []types.JobDescription{ {Flag: types.CronCapability, NodeType: types.WorkerNode}, {Flag: types.CustomComputeCapability, NodeType: types.WorkerNode}, {Flag: types.OCR3Capability, NodeType: types.WorkerNode}, + {Flag: types.GatewayDON, NodeType: types.GatewayDON}, + + // add more jobs as needed +} + +func checkForUnknownJobs(jobSpecs types.DonJobs) error { + for jobDesc := range jobSpecs { + found := false + for _, supportedJob := range SupportedJobs { + if jobDesc.Flag == supportedJob.Flag { + found = true + break + } + } + + if !found { + return errors.Errorf("unknown job type %s", jobDesc.Flag) + } + } + + return nil } func Create(offChainClient deployment.OffchainClient, don *devenv.DON, flags []string, jobSpecs types.DonJobs) error { - errCh := make(chan error, calculateJobCount(jobSpecs)) + if len(jobSpecs) == 0 { + return nil + } + if unknownErr := checkForUnknownJobs(jobSpecs); unknownErr != nil { + return errors.Wrap(unknownErr, "failed to create jobs") + } + + errCh := make(chan error, calculateJobCount(jobSpecs)) var wg sync.WaitGroup for _, jobDesc := range SupportedJobs { @@ -51,7 +79,11 @@ func Create(offChainClient deployment.OffchainClient, don *devenv.DON, flags []s var finalErr error for err := range errCh { - finalErr = errors.Wrap(finalErr, err.Error()) + if finalErr == nil { + finalErr = err + } else { + finalErr = errors.Wrap(finalErr, err.Error()) + } } if finalErr != nil { diff --git a/system-tests/lib/cre/don/jobs/por/por.go b/system-tests/lib/cre/don/jobs/por/por.go index 44ef67a91cf..5f3b58d9712 100644 --- a/system-tests/lib/cre/don/jobs/por/por.go +++ b/system-tests/lib/cre/don/jobs/por/por.go @@ -6,63 +6,137 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/pkg/errors" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" + "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" - "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" libc "github.com/smartcontractkit/chainlink/system-tests/lib/conversions" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" - keystoneflags "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" + creflags "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) -// If we wanted to by fancy we could also accept map[JobDescription]string that would get us the job spec -// if there's no job spec for the given JobDescription we would use the standard one, that could be easier -// than having to define the job spec for each JobDescription manually, in case someone wants to change one parameter -func GenerateJobSpecs(input types.GeneratePoRJobSpecsInput) (types.DonJobs, error) { +func GenerateJobSpecs(input *types.GeneratePoRJobSpecsInput) (types.DonsToJobSpecs, error) { + if input == nil { + return nil, errors.New("input is nil") + } if err := input.Validate(); err != nil { return nil, errors.Wrap(err, "input validation failed") } + donToJobSpecs := make(types.DonsToJobSpecs) + + gatewayConnectorData := input.GatewayConnectorOutput + + // we need to iterate over all DONs to see which need gateway connector and create a map of Don IDs and ETH addresses (which identify nodes that can use the connector) + // This map will be used to configure the gateway job on the node that runs it. Ccurrently, we support only a single gateway connector, even if CRE supports multiple + for _, donWithMetadata := range input.DonsWithMetadata { + // if it's a workflow DON or it has custom compute capability, it needs access to gateway connector + if creflags.HasFlag(donWithMetadata.Flags, types.WorkflowDON) || creflags.HasFlag(donWithMetadata.Flags, types.CustomComputeCapability) { + workflowNodeSet, err := node.FindManyWithLabel(donWithMetadata.NodesMetadata, &types.Label{Key: node.NodeTypeKey, Value: types.WorkerNode}, node.EqualLabels) + if err != nil { + return nil, errors.Wrap(err, "failed to find worker nodes") + } + + ethAddresses := make([]string, len(workflowNodeSet)) + var ethAddressErr error + for i, n := range workflowNodeSet { + ethAddresses[i], ethAddressErr = node.FindLabelValue(n, node.EthAddressKey) + if ethAddressErr != nil { + return nil, errors.Wrap(ethAddressErr, "failed to get eth address from labels") + } + } + gatewayConnectorData.Dons = append(gatewayConnectorData.Dons, types.GatewayConnectorDons{ + MembersEthAddresses: ethAddresses, + ID: donWithMetadata.DonMetadata.ID, + }) + } + } + + for _, donWithMetadata := range input.DonsWithMetadata { + jobSpecs, err := generateDonJobSpecs( + input.BlockchainOutput, + donWithMetadata, + input.OCR3CapabilityAddress, + input.CronCapBinName, + input.ExtraAllowedPorts, + input.ExtraAllowedIPs, + gatewayConnectorData, + ) + if err != nil { + return nil, errors.Wrapf(err, "failed to generate job specs for don %d", donWithMetadata.DonMetadata.ID) + } + + donToJobSpecs[donWithMetadata.DonMetadata.ID] = jobSpecs + } + + return donToJobSpecs, nil +} + +// If we wanted to by fancy we could also accept map[JobDescription]string that would get us the job spec +// if there's no job spec for the given JobDescription we would use the standard one, that could be easier +// than having to define the job spec for each JobDescription manually, in case someone wants to change one parameter +func generateDonJobSpecs( + blockchainOutput *blockchain.Output, + donWithMetadata *types.DonWithMetadata, + oCR3CapabilityAddress common.Address, + cronCapBinName string, + extraAllowedPorts []int, + extraAllowedIPs []string, + gatewayConnectorOutput types.GatewayConnectorOutput, +) (types.DonJobs, error) { jobSpecs := make(types.DonJobs) - chainIDInt, err := strconv.Atoi(input.BlockchainOutput.ChainID) + chainIDInt, err := strconv.Atoi(blockchainOutput.ChainID) if err != nil { return nil, errors.Wrap(err, "failed to convert chain ID to int") } chainIDUint64 := libc.MustSafeUint64(int64(chainIDInt)) - bootstrapNode, err := node.FindOneWithLabel(input.Don, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.BootstrapNode)}) + // create job specs for the gateway node + if creflags.HasFlag(donWithMetadata.Flags, types.GatewayDON) { + gatewayNode, nodeErr := node.FindOneWithLabel(donWithMetadata.NodesMetadata, &types.Label{Key: node.ExtraRolesKey, Value: types.GatewayNode}, node.LabelContains) + if nodeErr != nil { + return nil, errors.Wrap(nodeErr, "failed to find bootstrap node") + } + + gatewayNodeID, gatewayErr := node.FindLabelValue(gatewayNode, node.NodeIDKey) + if gatewayErr != nil { + return nil, errors.Wrap(gatewayErr, "failed to get gateway node id from labels") + } + + jobSpecs[types.JobDescription{Flag: types.GatewayDON, NodeType: types.GatewayNode}] = []*jobv1.ProposeJobRequest{jobs.AnyGateway(gatewayNodeID, chainIDUint64, donWithMetadata.ID, extraAllowedPorts, extraAllowedIPs, gatewayConnectorOutput)} + } + + // if it's only a gateway node, we don't need to create any other job specs + if creflags.HasOnlyOneFlag(donWithMetadata.Flags, types.GatewayDON) { + return jobSpecs, nil + } + + // look for boostrap node and then for required values in its labels + bootstrapNode, err := node.FindOneWithLabel(donWithMetadata.NodesMetadata, &types.Label{Key: node.NodeTypeKey, Value: types.BootstrapNode}, node.EqualLabels) if err != nil { return nil, errors.Wrap(err, "failed to find bootstrap node") } - donBootstrapNodePeerID, err := node.ToP2PID(*bootstrapNode, node.KeyExtractingTransformFn) + donBootstrapNodePeerID, err := node.ToP2PID(bootstrapNode, node.KeyExtractingTransformFn) if err != nil { return nil, errors.Wrap(err, "failed to get bootstrap node peer ID") } - var donBootstrapNodeHost string - for _, label := range bootstrapNode.Labels() { - if label.Key == node.HostLabelKey { - donBootstrapNodeHost = *label.Value - break - } - } - - if donBootstrapNodeHost == "" { - return nil, errors.New("failed to get bootstrap node host from labels") + donBootstrapNodeHost, hostErr := node.FindLabelValue(bootstrapNode, node.HostLabelKey) + if hostErr != nil { + return nil, errors.Wrap(hostErr, "failed to get bootstrap node host from labels") } - // configuration of bootstrap node - if keystoneflags.HasFlag(input.Flags, types.OCR3Capability) { - jobSpecs[types.JobDescription{Flag: types.OCR3Capability, NodeType: types.BootstrapNode}] = []*jobv1.ProposeJobRequest{jobs.BootstrapOCR3(bootstrapNode.NodeID, input.OCR3CapabilityAddress, chainIDUint64)} + bootstrapNodeID, nodeIDErr := node.FindLabelValue(bootstrapNode, node.NodeIDKey) + if nodeIDErr != nil { + return nil, errors.Wrap(nodeIDErr, "failed to get bootstrap node id from labels") } - // if it's a workflow DON or it has custom compute capability, we need to create a gateway job - if keystoneflags.HasFlag(input.Flags, types.WorkflowDON) || keystoneflags.HasFlag(input.Flags, types.CustomComputeCapability) { - jobSpecs[types.JobDescription{Flag: types.WorkflowDON, NodeType: types.BootstrapNode}] = []*jobv1.ProposeJobRequest{jobs.BootstrapGateway(input.Don, chainIDUint64, input.DonID, input.ExtraAllowedPorts, input.ExtraAllowedIPs, input.GatewayConnectorOutput)} + // create job specs for the bootstrap node + if creflags.HasFlag(donWithMetadata.Flags, types.OCR3Capability) { + jobSpecs[types.JobDescription{Flag: types.OCR3Capability, NodeType: types.BootstrapNode}] = []*jobv1.ProposeJobRequest{jobs.BootstrapOCR3(bootstrapNodeID, oCR3CapabilityAddress, chainIDUint64)} } ocrPeeringData := types.OCRPeeringData{ @@ -71,15 +145,21 @@ func GenerateJobSpecs(input types.GeneratePoRJobSpecsInput) (types.DonJobs, erro Port: 5001, } - workflowNodeSet, err := node.FindManyWithLabel(input.Don, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.WorkerNode)}) + // create job specs for the worker nodes + workflowNodeSet, err := node.FindManyWithLabel(donWithMetadata.NodesMetadata, &types.Label{Key: node.NodeTypeKey, Value: types.WorkerNode}, node.EqualLabels) if err != nil { + // there should be no DON without worker nodes, even gateway DON is composed of a single worker node return nil, errors.Wrap(err, "failed to find worker nodes") } - // configuration of worker nodes - for _, node := range workflowNodeSet { - if keystoneflags.HasFlag(input.Flags, types.CronCapability) { - jobSpec := jobs.WorkerStandardCapability(node.NodeID, "cron-capabilities", jobs.ExternalCapabilityPath(input.CronCapBinName), jobs.EmptyStdCapConfig) + for _, workerNode := range workflowNodeSet { + nodeID, nodeIDErr := node.FindLabelValue(workerNode, node.NodeIDKey) + if nodeIDErr != nil { + return nil, errors.Wrap(nodeIDErr, "failed to get node id from labels") + } + + if creflags.HasFlag(donWithMetadata.Flags, types.CronCapability) { + jobSpec := jobs.WorkerStandardCapability(nodeID, "cron-capability", jobs.ExternalCapabilityPath(cronCapBinName), jobs.EmptyStdCapConfig) jobDesc := types.JobDescription{Flag: types.CronCapability, NodeType: types.WorkerNode} if _, ok := jobSpecs[jobDesc]; !ok { @@ -89,7 +169,7 @@ func GenerateJobSpecs(input types.GeneratePoRJobSpecsInput) (types.DonJobs, erro } } - if keystoneflags.HasFlag(input.Flags, types.CustomComputeCapability) { + if creflags.HasFlag(donWithMetadata.Flags, types.CustomComputeCapability) { config := `""" NumWorkers = 3 [rateLimiter] @@ -99,7 +179,7 @@ func GenerateJobSpecs(input types.GeneratePoRJobSpecsInput) (types.DonJobs, erro perSenderBurst = 5 """` - jobSpec := jobs.WorkerStandardCapability(node.NodeID, "custom-compute", "__builtin_custom-compute-action", config) + jobSpec := jobs.WorkerStandardCapability(nodeID, "custom-compute", "__builtin_custom-compute-action", config) jobDesc := types.JobDescription{Flag: types.CustomComputeCapability, NodeType: types.WorkerNode} if _, ok := jobSpecs[jobDesc]; !ok { @@ -109,8 +189,18 @@ func GenerateJobSpecs(input types.GeneratePoRJobSpecsInput) (types.DonJobs, erro } } - if keystoneflags.HasFlag(input.Flags, types.OCR3Capability) { - jobSpec := jobs.WorkerOCR3(node.NodeID, input.OCR3CapabilityAddress, common.HexToAddress(node.AccountAddr[chainIDUint64]), node.Ocr2KeyBundleID, ocrPeeringData, chainIDUint64) + nodeEthAddr, ethErr := node.FindLabelValue(workerNode, node.EthAddressKey) + if ethErr != nil { + return nil, errors.Wrap(ethErr, "failed to get eth address from labels") + } + + ocr2KeyBundleID, ocr2Err := node.FindLabelValue(workerNode, node.NodeOCR2KeyBundleIDKey) + if ocr2Err != nil { + return nil, errors.Wrap(ocr2Err, "failed to get ocr2 key bundle id from labels") + } + + if creflags.HasFlag(donWithMetadata.Flags, types.OCR3Capability) { + jobSpec := jobs.WorkerOCR3(nodeID, oCR3CapabilityAddress, nodeEthAddr, ocr2KeyBundleID, ocrPeeringData, chainIDUint64) jobDesc := types.JobDescription{Flag: types.OCR3Capability, NodeType: types.WorkerNode} if _, ok := jobSpecs[jobDesc]; !ok { diff --git a/system-tests/lib/cre/don/node/node.go b/system-tests/lib/cre/don/node/node.go index 70f1c2be72c..629a0882dfb 100644 --- a/system-tests/lib/cre/don/node/node.go +++ b/system-tests/lib/cre/don/node/node.go @@ -7,7 +7,6 @@ import ( "strconv" "strings" - "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" @@ -15,6 +14,17 @@ import ( ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" ) +const ( + NodeTypeKey = "type" + HostLabelKey = "host" + IndexKey = "node_index" + EthAddressKey = "eth_address" + ExtraRolesKey = "extra_roles" + NodeIDKey = "node_id" + NodeOCR2KeyBundleIDKey = "ocr2_key_bundle_id" + NodeP2PIDKey = "p2p_id" +) + type stringTransformer func(string) string func NoOpTransformFn(value string) string { @@ -29,28 +39,22 @@ func KeyExtractingTransformFn(value string) string { return value } -func ToP2PID(node devenv.Node, transformFn stringTransformer) (string, error) { - for _, label := range node.Labels() { - if label.Key == devenv.NodeLabelP2PIDType { - if label.Value == nil { - return "", fmt.Errorf("p2p label value is nil for node %s", node.Name) +func ToP2PID(node *types.NodeMetadata, transformFn stringTransformer) (string, error) { + for _, label := range node.Labels { + if label.Key == NodeP2PIDKey { + if label.Value == "" { + return "", errors.New("p2p label value is empty for node") } - return transformFn(*label.Value), nil + return transformFn(label.Value), nil } } - return "", fmt.Errorf("p2p label not found for node %s", node.Name) + return "", errors.New("p2p label not found for node") } -const ( - RoleLabelKey = "role" - HostLabelKey = "host" - NodeIndexKey = "node_index" -) - // copied from Bala's unmerged PR: https://github.com/smartcontractkit/chainlink/pull/15751 // TODO: remove this once the PR is merged and import his function -// IMPORTANT ADDITION: prefix to differentiate between the different DONs +// IMPORTANT ADDITION: prefix to differentiate between the different DONs func GetNodeInfo(nodeOut *ns.Output, prefix string, bootstrapNodeCount int) ([]devenv.NodeInfo, error) { var nodeInfo []devenv.NodeInfo for i := 1; i <= len(nodeOut.CLNodes); i++ { @@ -71,8 +75,8 @@ func GetNodeInfo(nodeOut *ns.Output, prefix string, bootstrapNodeCount int) ([]d }, Labels: map[string]string{ HostLabelKey: nodeOut.CLNodes[i-1].Node.ContainerName, - NodeIndexKey: strconv.Itoa(i - 1), - RoleLabelKey: types.BootstrapNode, + IndexKey: strconv.Itoa(i - 1), + NodeTypeKey: types.BootstrapNode, }, }) } else { @@ -88,8 +92,8 @@ func GetNodeInfo(nodeOut *ns.Output, prefix string, bootstrapNodeCount int) ([]d }, Labels: map[string]string{ HostLabelKey: nodeOut.CLNodes[i-1].Node.ContainerName, - NodeIndexKey: strconv.Itoa(i - 1), - RoleLabelKey: types.WorkerNode, + IndexKey: strconv.Itoa(i - 1), + NodeTypeKey: types.WorkerNode, }, }) } @@ -97,48 +101,61 @@ func GetNodeInfo(nodeOut *ns.Output, prefix string, bootstrapNodeCount int) ([]d return nodeInfo, nil } -func FindOneWithLabel(nodes *devenv.DON, wantedLabel *ptypes.Label) (*devenv.Node, error) { +func FindOneWithLabel(nodes []*types.NodeMetadata, wantedLabel *types.Label, labelMatcherFn labelMatcherFn) (*types.NodeMetadata, error) { if wantedLabel == nil { return nil, errors.New("label is nil") } - for _, node := range nodes.Nodes { - for _, label := range node.Labels() { - if wantedLabel.Key == label.Key && equalLabels(wantedLabel.Value, label.Value) { - return &node, nil + for _, node := range nodes { + for _, label := range node.Labels { + if wantedLabel.Key == label.Key && labelMatcherFn(wantedLabel.Value, label.Value) { + return node, nil } } } - return nil, fmt.Errorf("node with label %s=%s not found", wantedLabel.Key, *wantedLabel.Value) + return nil, fmt.Errorf("node with label %s=%s not found", wantedLabel.Key, wantedLabel.Value) } -func FindManyWithLabel(nodes *devenv.DON, wantedLabel *ptypes.Label) ([]*devenv.Node, error) { +func FindManyWithLabel(nodes []*types.NodeMetadata, wantedLabel *types.Label, labelMatcherFn labelMatcherFn) ([]*types.NodeMetadata, error) { if wantedLabel == nil { return nil, errors.New("label is nil") } - var foundNodes []*devenv.Node + var foundNodes []*types.NodeMetadata - for _, node := range nodes.Nodes { - for _, label := range node.Labels() { - if wantedLabel.Key == label.Key && equalLabels(wantedLabel.Value, label.Value) { - foundNodes = append(foundNodes, &node) + for _, node := range nodes { + for _, label := range node.Labels { + if wantedLabel.Key == label.Key && labelMatcherFn(wantedLabel.Value, label.Value) { + foundNodes = append(foundNodes, node) } } } if len(foundNodes) == 0 { - return nil, fmt.Errorf("node with label %s=%s not found", wantedLabel.Key, *wantedLabel.Value) + return nil, fmt.Errorf("node with label %s=%s not found", wantedLabel.Key, wantedLabel.Value) } return foundNodes, nil } -func equalLabels(first, second *string) bool { - if first == nil && second == nil { - return true - } - if first == nil || second == nil { - return false +func FindLabelValue(node *types.NodeMetadata, labelKey string) (string, error) { + for _, label := range node.Labels { + if label.Key == labelKey { + if label.Value == "" { + return "", fmt.Errorf("label %s found, but its value is empty", labelKey) + } + return label.Value, nil + } } - return *first == *second + + return "", fmt.Errorf("label %s not found", labelKey) +} + +type labelMatcherFn func(first, second string) bool + +func EqualLabels(first, second string) bool { + return first == second +} + +func LabelContains(first, second string) bool { + return strings.Contains(first, second) } diff --git a/system-tests/lib/cre/don/peering.go b/system-tests/lib/cre/don/peering.go index 0cac7ce5d51..c95ccef4c9d 100644 --- a/system-tests/lib/cre/don/peering.go +++ b/system-tests/lib/cre/don/peering.go @@ -3,60 +3,48 @@ package don import ( "github.com/pkg/errors" - "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" - "github.com/smartcontractkit/chainlink-testing-framework/lib/utils/ptr" - "github.com/smartcontractkit/chainlink/deployment/environment/devenv" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) -func globalBootstraperNodeData(donTopologies []*types.DonWithMetadata) (string, string, error) { - var findHost = func(n devenv.Node) string { - for _, label := range n.Labels() { - if label.Key == node.HostLabelKey { - return *label.Value - } - } - return "" - } - - if len(donTopologies) == 1 { - bootstrapNode, err := node.FindOneWithLabel(donTopologies[0].DON, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.BootstrapNode)}) +func globalBootstraperNodeData(topology *types.Topology) (string, string, error) { + if len(topology.DonsMetadata) == 1 { + bootstrapNode, err := node.FindOneWithLabel(topology.DonsMetadata[0].NodesMetadata, &types.Label{Key: node.NodeTypeKey, Value: types.BootstrapNode}, node.EqualLabels) if err != nil { return "", "", errors.Wrap(err, "failed to find bootstrap node") } // if there is only one DON, then the global bootstrapper is the bootstrap node of the DON - peerID, err := node.ToP2PID(*bootstrapNode, node.KeyExtractingTransformFn) + peerID, err := node.ToP2PID(bootstrapNode, node.KeyExtractingTransformFn) if err != nil { - return "", "", errors.Wrapf(err, "failed to get peer ID for node %s", donTopologies[0].DON.Nodes[0].Name) + return "", "", errors.Wrap(err, "failed to get peer ID for the bootstrap node") } - bootstrapNodeHost := findHost(*bootstrapNode) - if bootstrapNodeHost == "" { - return "", "", errors.New("failed to get bootstrap node host from labels") + bootstrapNodeHost, hostErr := node.FindLabelValue(bootstrapNode, node.HostLabelKey) + if hostErr != nil { + return "", "", errors.Wrap(hostErr, "failed to get bootstrap node host from labels") } return peerID, bootstrapNodeHost, nil - } else if len(donTopologies) > 1 { + } else if len(topology.DonsMetadata) > 1 { // if there's more than one DON, then peering capabilitity needs to point to the same bootstrap node // for all the DONs, and so we need to find it first. For us, it will always be the bootstrap node of the workflow DON. - for _, donTopology := range donTopologies { + for _, donTopology := range topology.DonsMetadata { if flags.HasFlag(donTopology.Flags, types.WorkflowDON) { - bootstrapNode, err := node.FindOneWithLabel(donTopology.DON, &ptypes.Label{Key: node.RoleLabelKey, Value: ptr.Ptr(types.BootstrapNode)}) + bootstrapNode, err := node.FindOneWithLabel(donTopology.NodesMetadata, &types.Label{Key: node.NodeTypeKey, Value: types.BootstrapNode}, node.EqualLabels) if err != nil { return "", "", errors.Wrap(err, "failed to find bootstrap node") } - peerID, err := node.ToP2PID(*bootstrapNode, node.KeyExtractingTransformFn) + peerID, err := node.ToP2PID(bootstrapNode, node.KeyExtractingTransformFn) if err != nil { - return "", "", errors.Wrapf(err, "failed to get peer ID for node %s", bootstrapNode.Name) + return "", "", errors.Wrapf(err, "failed to get peer ID for workernode %s", "CHANGE ME") } - bootstrapNodeHost := findHost(*bootstrapNode) - if bootstrapNodeHost == "" { - return "", "", errors.New("failed to get bootstrap node host from labels") + bootstrapNodeHost, hostErr := node.FindLabelValue(bootstrapNode, node.HostLabelKey) + if hostErr != nil { + return "", "", errors.Wrap(hostErr, "failed to get bootstrap node host from labels") } return peerID, bootstrapNodeHost, nil @@ -69,13 +57,13 @@ func globalBootstraperNodeData(donTopologies []*types.DonWithMetadata) (string, return "", "", errors.New("expected at least one DON topology") } -func FindPeeringData(donTopologies []*types.DonWithMetadata) (types.PeeringData, error) { +func FindPeeringData(donTopologies *types.Topology) (types.CapabilitiesPeeringData, error) { globalBootstraperPeerID, globalBootstraperHost, err := globalBootstraperNodeData(donTopologies) if err != nil { - return types.PeeringData{}, err + return types.CapabilitiesPeeringData{}, err } - return types.PeeringData{ + return types.CapabilitiesPeeringData{ GlobalBootstraperPeerID: globalBootstraperPeerID, GlobalBootstraperHost: globalBootstraperHost, }, nil diff --git a/system-tests/lib/cre/don/secrets/secrets.go b/system-tests/lib/cre/don/secrets/secrets.go new file mode 100644 index 00000000000..cbc6a9ecdf6 --- /dev/null +++ b/system-tests/lib/cre/don/secrets/secrets.go @@ -0,0 +1,50 @@ +package secrets + +import ( + "github.com/pelletier/go-toml/v2" + "github.com/pkg/errors" + + cretypes "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" + "github.com/smartcontractkit/chainlink/system-tests/lib/types" +) + +func GenerateSecrets(input *cretypes.GenerateSecretsInput) (cretypes.NodeIndexToSecretsOverride, error) { + if input == nil { + return nil, errors.New("input is nil") + } + if err := input.Validate(); err != nil { + return nil, errors.Wrap(err, "input validation failed") + } + + overrides := make(cretypes.NodeIndexToSecretsOverride) + + for i := range input.DonMetadata.NodesMetadata { + nodeSecret := types.NodeSecret{} + if input.EVMKeys != nil { + nodeSecret.EthKeys = types.NodeEthKeyWrapper{} + for _, chainID := range input.EVMKeys.ChainIDs { + nodeSecret.EthKeys.EthKeys = append(nodeSecret.EthKeys.EthKeys, types.NodeEthKey{ + JSON: string(input.EVMKeys.EncryptedJSONs[i]), + Password: input.EVMKeys.Password, + ChainID: chainID, + }) + } + } + + if input.P2PKeys != nil { + nodeSecret.P2PKey = types.NodeP2PKey{ + JSON: string(input.P2PKeys.EncryptedJSONs[i]), + Password: input.P2PKeys.Password, + } + } + + nodeSecretString, err := toml.Marshal(nodeSecret) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal node secrets") + } + + overrides[i] = string(nodeSecretString) + } + + return overrides, nil +} diff --git a/system-tests/lib/cre/environment/environment.go b/system-tests/lib/cre/environment/environment.go index 9d48bb179ea..b28b129526f 100644 --- a/system-tests/lib/cre/environment/environment.go +++ b/system-tests/lib/cre/environment/environment.go @@ -7,47 +7,39 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc/credentials/insecure" - "github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd" "github.com/smartcontractkit/chainlink-testing-framework/seth" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" - libdon "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don" libnode "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node" "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) -func BuildTopologyAndCLDEnvironment(lgr logger.Logger, nodeSetInput []*types.CapabilitiesAwareNodeSet, jdOutput *jd.Output, nodeSetOutput []*types.WrappedNodeOutput, blockchainOutput *blockchain.Output, sethClient *seth.Client) (*deployment.Environment, *types.DonTopology, error) { - env, dons, err := buildChainlinkDeploymentEnv(lgr, jdOutput, nodeSetOutput, blockchainOutput, sethClient) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to build chainlink deployment environment") +func BuildFullCLDEnvironment(lgr logger.Logger, input *types.FullCLDEnvironmentInput) (*types.FullCLDEnvironmentOutput, error) { + if input == nil { + return nil, errors.New("input is nil") } - donTopology, err := libdon.BuildDONTopology(dons, nodeSetInput, nodeSetOutput) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to build DON topology") + if err := input.Validate(); err != nil { + return nil, errors.Wrap(err, "input validation failed") } - return env, donTopology, nil -} + envs := make([]*deployment.Environment, len(input.NodeSetOutput)) + dons := make([]*devenv.DON, len(input.NodeSetOutput)) -func buildChainlinkDeploymentEnv(lgr logger.Logger, jdOutput *jd.Output, nodeSetOutput []*types.WrappedNodeOutput, blockchainOutput *blockchain.Output, sethClient *seth.Client) (*deployment.Environment, []*devenv.DON, error) { - envs := make([]*deployment.Environment, len(nodeSetOutput)) - dons := make([]*devenv.DON, len(nodeSetOutput)) var allNodesInfo []devenv.NodeInfo - for i, nodeOutput := range nodeSetOutput { + for i, nodeOutput := range input.NodeSetOutput { // assume that each nodeset has only one bootstrap node nodeInfo, err := libnode.GetNodeInfo(nodeOutput.Output, nodeOutput.NodeSetName, 1) if err != nil { - return nil, nil, errors.Wrap(err, "failed to get node info") + return nil, errors.Wrap(err, "failed to get node info") } allNodesInfo = append(allNodesInfo, nodeInfo...) jdConfig := devenv.JDConfig{ - GRPC: jdOutput.HostGRPCUrl, - WSRPC: jdOutput.DockerWSRPCUrl, + GRPC: input.JdOutput.HostGRPCUrl, + WSRPC: input.JdOutput.DockerWSRPCUrl, Creds: insecure.NewCredentials(), NodeInfo: nodeInfo, } @@ -56,25 +48,25 @@ func buildChainlinkDeploymentEnv(lgr logger.Logger, jdOutput *jd.Output, nodeSet JDConfig: jdConfig, Chains: []devenv.ChainConfig{ { - ChainID: sethClient.Cfg.Network.ChainID, - ChainName: sethClient.Cfg.Network.Name, - ChainType: strings.ToUpper(blockchainOutput.Family), + ChainID: input.SethClient.Cfg.Network.ChainID, + ChainName: input.SethClient.Cfg.Network.Name, + ChainType: strings.ToUpper(input.BlockchainOutput.Family), WSRPCs: []devenv.CribRPCs{{ - External: blockchainOutput.Nodes[0].HostWSUrl, - Internal: blockchainOutput.Nodes[0].DockerInternalWSUrl, + External: input.BlockchainOutput.Nodes[0].HostWSUrl, + Internal: input.BlockchainOutput.Nodes[0].DockerInternalWSUrl, }}, HTTPRPCs: []devenv.CribRPCs{{ - External: blockchainOutput.Nodes[0].HostHTTPUrl, - Internal: blockchainOutput.Nodes[0].DockerInternalHTTPUrl, + External: input.BlockchainOutput.Nodes[0].HostHTTPUrl, + Internal: input.BlockchainOutput.Nodes[0].DockerInternalHTTPUrl, }}, - DeployerKey: sethClient.NewTXOpts(seth.WithNonce(nil)), // set nonce to nil, so that it will be fetched from the chain + DeployerKey: input.SethClient.NewTXOpts(seth.WithNonce(nil)), // set nonce to nil, so that it will be fetched from the chain }, }, } env, don, err := devenv.NewEnvironment(context.Background, lgr, devenvConfig) if err != nil { - return nil, nil, errors.Wrap(err, "failed to create environment") + return nil, errors.Wrap(err, "failed to create environment") } envs[i] = env @@ -86,30 +78,63 @@ func buildChainlinkDeploymentEnv(lgr logger.Logger, jdOutput *jd.Output, nodeSet nodeIDs = append(nodeIDs, env.NodeIDs...) } + for i, don := range dons { + for j, node := range input.Topology.DonsMetadata[i].NodesMetadata { + // both are required for job creation + node.Labels = append(node.Labels, &types.Label{ + Key: libnode.NodeIDKey, + Value: don.NodeIds()[j], + }) + + node.Labels = append(node.Labels, &types.Label{ + Key: libnode.NodeOCR2KeyBundleIDKey, + Value: don.Nodes[j].Ocr2KeyBundleID, + }) + + node.Labels = append(node.Labels, &types.Label{ + Key: libnode.NodeOCR2KeyBundleIDKey, + Value: don.Nodes[j].Ocr2KeyBundleID, + }) + } + } + // Create a JD client that can interact with all the nodes, otherwise if it has node IDs of nodes that belong only to one Don, // it will still propose jobs to unknown nodes, but won't accept them automatically jd, err := devenv.NewJDClient(context.Background(), devenv.JDConfig{ - GRPC: jdOutput.HostGRPCUrl, - WSRPC: jdOutput.DockerWSRPCUrl, + GRPC: input.JdOutput.HostGRPCUrl, + WSRPC: input.JdOutput.DockerWSRPCUrl, Creds: insecure.NewCredentials(), NodeInfo: allNodesInfo, }) if err != nil { - return nil, nil, errors.Wrap(err, "failed to create JD client") + return nil, errors.Wrap(err, "failed to create JD client") } // we assume that all DONs run on the same chain and that there's only one chain - // also, we don't care which instance of offchain client we use, because we have - // only one instance of offchain client and we have just configured it to work - // with nodes from all DONs - return &deployment.Environment{ - Name: envs[0].Name, - Logger: envs[0].Logger, - ExistingAddresses: envs[0].ExistingAddresses, - Chains: envs[0].Chains, - Offchain: jd, - OCRSecrets: envs[0].OCRSecrets, - GetContext: envs[0].GetContext, - NodeIDs: nodeIDs, - }, dons, nil + output := &types.FullCLDEnvironmentOutput{ + Environment: &deployment.Environment{ + Name: envs[0].Name, + Logger: envs[0].Logger, + ExistingAddresses: input.ExistingAddresses, + Chains: envs[0].Chains, + Offchain: jd, + OCRSecrets: envs[0].OCRSecrets, + GetContext: envs[0].GetContext, + NodeIDs: nodeIDs, + }, + } + + donTopology := &types.DonTopology{} + donTopology.WorkflowDonID = input.Topology.WorkflowDONID + + for i, donMetadata := range input.Topology.DonsMetadata { + donTopology.DonsWithMetadata = append(donTopology.DonsWithMetadata, &types.DonWithMetadata{ + DON: dons[i], + DonMetadata: donMetadata, + }) + } + + output.DonTopology = donTopology + + return output, nil } diff --git a/system-tests/lib/cre/flags/flags.go b/system-tests/lib/cre/flags/flags.go index 5bedcb8fd95..319aa551b00 100644 --- a/system-tests/lib/cre/flags/flags.go +++ b/system-tests/lib/cre/flags/flags.go @@ -8,8 +8,8 @@ import ( "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" ) -func DONTopologyWithFlag(donTopologies []*types.DonWithMetadata, flag string) []*types.DonWithMetadata { - var result []*types.DonWithMetadata +func DonMetadataWithFlag(donTopologies []*types.DonMetadata, flag string) []*types.DonMetadata { + var result []*types.DonMetadata for _, donTopology := range donTopologies { if HasFlag(donTopology.Flags, flag) { @@ -24,8 +24,12 @@ func HasFlag(values []string, flag string) bool { return slices.Contains(values, flag) } -func OneDONTopologyWithFlag(donTopologies []*types.DonWithMetadata, flag string) (*types.DonWithMetadata, error) { - donTopologies = DONTopologyWithFlag(donTopologies, flag) +func HasOnlyOneFlag(values []string, flag string) bool { + return slices.Contains(values, flag) && len(values) == 1 +} + +func OneDonMetadataWithFlag(donTopologies []*types.DonMetadata, flag string) (*types.DonMetadata, error) { + donTopologies = DonMetadataWithFlag(donTopologies, flag) if len(donTopologies) != 1 { return nil, errors.Errorf("expected exactly one DON topology with flag %s, got %d", flag, len(donTopologies)) } @@ -35,11 +39,11 @@ func OneDONTopologyWithFlag(donTopologies []*types.DonWithMetadata, flag string) func NodeSetFlags(nodeSet *types.CapabilitiesAwareNodeSet) ([]string, error) { var stringCaps []string - if len(nodeSet.Capabilities) == 0 && nodeSet.DONType == "" { + if len(nodeSet.Capabilities) == 0 && len(nodeSet.DONTypes) == 0 { // if no flags are set, we assign all known capabilities to the DON return types.SingleDonFlags, nil } - stringCaps = append(stringCaps, append(nodeSet.Capabilities, nodeSet.DONType)...) + stringCaps = append(stringCaps, append(nodeSet.Capabilities, nodeSet.DONTypes...)...) return stringCaps, nil } diff --git a/system-tests/lib/cre/types/flags.go b/system-tests/lib/cre/types/flags.go index 63e6de7d2bd..4d1855f4d00 100644 --- a/system-tests/lib/cre/types/flags.go +++ b/system-tests/lib/cre/types/flags.go @@ -6,6 +6,7 @@ type CapabilityFlag = string const ( WorkflowDON CapabilityFlag = "workflow" CapabilitiesDON CapabilityFlag = "capabilities" + GatewayDON CapabilityFlag = "gateway" ) // Capabilities @@ -20,5 +21,5 @@ const ( var ( // Add new capabilities here as well, if single DON should have them by default - SingleDonFlags = []string{"workflow", "capabilities", "ocr3", "cron", "custom-compute", "write-evm"} + SingleDonFlags = []string{"capabilities", "ocr3", "cron", "custom-compute", "write-evm"} ) diff --git a/system-tests/lib/cre/types/keystone.go b/system-tests/lib/cre/types/keystone.go index 6ca3929d58f..8d611bc4f74 100644 --- a/system-tests/lib/cre/types/keystone.go +++ b/system-tests/lib/cre/types/keystone.go @@ -6,12 +6,14 @@ import ( "github.com/ethereum/go-ethereum/common" jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" + "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" "github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/framework/components/jd" ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set" "github.com/smartcontractkit/chainlink-testing-framework/seth" "github.com/smartcontractkit/chainlink/deployment" "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/smartcontractkit/chainlink/system-tests/lib/types" ) type NodeType = string @@ -19,6 +21,7 @@ type NodeType = string const ( BootstrapNode NodeType = "bootstrap" WorkerNode NodeType = "worker" + GatewayNode NodeType = "gateway" ) type JobDescription struct { @@ -34,8 +37,8 @@ type ConfigDescription struct { type DonJobs = map[JobDescription][]*jobv1.ProposeJobRequest type DonsToJobSpecs = map[uint32]DonJobs -type NodeIndexToConfigOverrides = map[int]string -type DonsToConfigOverrides = map[uint32]NodeIndexToConfigOverrides +type NodeIndexToConfigOverride = map[int]string +type NodeIndexToSecretsOverride = map[int]string type KeystoneContractsInput struct { ChainSelector uint64 `toml:"-"` @@ -157,56 +160,57 @@ type WrappedNodeOutput struct { Capabilities []string } -type ConfigureDonInput struct { - CldEnv *deployment.Environment - BlockchainOutput *blockchain.Output - DonTopology *DonTopology - JdOutput *jd.Output - DonToJobSpecs DonsToJobSpecs - DonToConfigOverrides DonsToConfigOverrides +type CreateJobsInput struct { + CldEnv *deployment.Environment + DonTopology *DonTopology + DonToJobSpecs DonsToJobSpecs } -func (c *ConfigureDonInput) Validate() error { +func (c *CreateJobsInput) Validate() error { if c.CldEnv == nil { return errors.New("chainlink deployment env not set") } - if c.BlockchainOutput == nil { - return errors.New("blockchain output not set") - } if c.DonTopology == nil { return errors.New("don topology not set") } - if len(c.DonTopology.MetaDons) == 0 { - return errors.New("meta dons not set") - } - if c.JdOutput == nil { - return errors.New("jd output not set") + if len(c.DonTopology.DonsWithMetadata) == 0 { + return errors.New("topology dons not set") } if len(c.DonToJobSpecs) == 0 { return errors.New("don to job specs not set") } - if len(c.DonToConfigOverrides) == 0 { - return errors.New("don to config overrides not set") - } return nil } -type ConfigureDonOutput struct { - JdOutput *deployment.OffchainClient -} - type DebugInput struct { - DonTopology *DonTopology + DebugDons []*DebugDon BlockchainOutput *blockchain.Output } +type DebugDon struct { + Flags []string + ContainerNames []string + NodesMetadata []*NodeMetadata +} + func (d *DebugInput) Validate() error { - if d.DonTopology == nil { + if d.DebugDons == nil { return errors.New("don topology not set") } - if len(d.DonTopology.MetaDons) == 0 { - return errors.New("meta dons not set") + if len(d.DebugDons) == 0 { + return errors.New("debug don not set") + } + for _, don := range d.DebugDons { + if len(don.ContainerNames) == 0 { + return errors.New("container names not set") + } + if len(don.NodesMetadata) == 0 { + return errors.New("nodes metadata not set") + } + if len(don.Flags) == 0 { + return errors.New("flags not set") + } } if d.BlockchainOutput == nil { return errors.New("blockchain output not set") @@ -217,7 +221,7 @@ func (d *DebugInput) Validate() error { type ConfigureKeystoneInput struct { ChainSelector uint64 - DonTopology *DonTopology + Topology *Topology CldEnv *deployment.Environment } @@ -225,10 +229,10 @@ func (c *ConfigureKeystoneInput) Validate() error { if c.ChainSelector == 0 { return errors.New("chain selector not set") } - if c.DonTopology == nil { + if c.Topology == nil { return errors.New("don topology not set") } - if len(c.DonTopology.MetaDons) == 0 { + if len(c.Topology.DonsMetadata) == 0 { return errors.New("meta dons not set") } if c.CldEnv == nil { @@ -238,19 +242,21 @@ func (c *ConfigureKeystoneInput) Validate() error { return nil } +type GatewayConnectorDons struct { + MembersEthAddresses []string + ID uint32 +} + type GatewayConnectorOutput struct { - Host string // do not set, it will be set dynamically + Dons []GatewayConnectorDons // do not set, it will be set dynamically + Host string // do not set, it will be set dynamically Path string Port int } type GeneratePoRJobSpecsInput struct { - CldEnv *deployment.Environment - Don *devenv.DON - NodeOutput *WrappedNodeOutput + DonsWithMetadata []*DonWithMetadata BlockchainOutput *blockchain.Output - DonID uint32 - Flags []string OCR3CapabilityAddress common.Address ExtraAllowedPorts []int ExtraAllowedIPs []string @@ -259,47 +265,34 @@ type GeneratePoRJobSpecsInput struct { } func (g *GeneratePoRJobSpecsInput) Validate() error { - if g.CldEnv == nil { - return errors.New("chainlink deployment env not set") - } - if g.Don == nil { - return errors.New("don not set") - } - if len(g.Don.Nodes) == 0 { - return errors.New("don nodes not set") - } - if g.NodeOutput == nil { - return errors.New("node output not set") + if len(g.DonsWithMetadata) == 0 { + return errors.New("metadata dons not set") } if g.BlockchainOutput == nil { return errors.New("blockchain output not set") } - if g.DonID == 0 { - return errors.New("don id not set") - } - if len(g.Flags) == 0 { - return errors.New("flags not set") - } if g.OCR3CapabilityAddress == (common.Address{}) { return errors.New("ocr3 capability address not set") } if g.CronCapBinName == "" { return errors.New("cron cap bin name not set") } - if g.GatewayConnectorOutput == (GatewayConnectorOutput{}) { - return errors.New("gateway connector output not set") + if g.GatewayConnectorOutput.Path == "" { + return errors.New("gateway connector path is not set") + } + if g.GatewayConnectorOutput.Port == 0 { + return errors.New("gateway connector port is not set") } return nil } type GeneratePoRConfigsInput struct { - Don *devenv.DON - NodeInput *CapabilitiesAwareNodeSet + DonMetadata *DonMetadata BlockchainOutput *blockchain.Output DonID uint32 Flags []string - PeeringData PeeringData + PeeringData CapabilitiesPeeringData CapabilitiesRegistryAddress common.Address WorkflowRegistryAddress common.Address ForwarderAddress common.Address @@ -307,15 +300,9 @@ type GeneratePoRConfigsInput struct { } func (g *GeneratePoRConfigsInput) Validate() error { - if g.Don == nil { - return errors.New("don not set") - } - if len(g.Don.Nodes) == 0 { + if len(g.DonMetadata.NodesMetadata) == 0 { return errors.New("don nodes not set") } - if g.NodeInput == nil { - return errors.New("node input not set") - } if g.BlockchainOutput == nil { return errors.New("blockchain output not set") } @@ -325,7 +312,7 @@ func (g *GeneratePoRConfigsInput) Validate() error { if len(g.Flags) == 0 { return errors.New("flags not set") } - if g.PeeringData == (PeeringData{}) { + if g.PeeringData == (CapabilitiesPeeringData{}) { return errors.New("peering data not set") } if g.CapabilitiesRegistryAddress == (common.Address{}) { @@ -344,27 +331,62 @@ func (g *GeneratePoRConfigsInput) Validate() error { return nil } -// DonWithMetadata is a struct that holds the DON references and various metadata +type ToplogyInput struct { + NodeSetInput []*CapabilitiesAwareNodeSet + DonToEthAddress map[uint32][]common.Address +} + type DonWithMetadata struct { - DON *devenv.DON - NodeInput *CapabilitiesAwareNodeSet - NodeOutput *WrappedNodeOutput - ID uint32 - Flags []string + DON *devenv.DON + *DonMetadata +} + +type DonMetadata struct { + NodesMetadata []*NodeMetadata + Flags []string + ID uint32 + Name string +} + +type Label struct { + Key string + Value string +} + +func LabelFromProto(p *ptypes.Label) (*Label, error) { + if p.Value == nil { + return nil, errors.New("value not set") + } + return &Label{ + Key: p.Key, + Value: *p.Value, + }, nil +} + +type NodeMetadata struct { + Labels []*Label +} + +type Topology struct { + WorkflowDONID uint32 + DonsMetadata []*DonMetadata + GatewayConnectorOutput *GatewayConnectorOutput } type DonTopology struct { - WorkflowDONID uint32 - MetaDons []*DonWithMetadata + WorkflowDonID uint32 + DonsWithMetadata []*DonWithMetadata } type CapabilitiesAwareNodeSet struct { *ns.Input - Capabilities []string `toml:"capabilities"` - DONType string `toml:"don_type"` + Capabilities []string + DONTypes []string + BootstrapNodeIndex int // -1 -> no bootstrap, only used if the DON doesn't hae the GatewayDON flag + GatewayNodeIndex int // -1 -> no gateway, only used if the DON has the GatewayDON flag } -type PeeringData struct { +type CapabilitiesPeeringData struct { GlobalBootstraperPeerID string GlobalBootstraperHost string Port int @@ -375,3 +397,97 @@ type OCRPeeringData struct { OCRBootstraperHost string Port int } + +type GenerateKeysInput struct { + GenerateEVMKeysForChainIDs []int + GenerateP2PKeys bool + Topology *Topology + Password string +} + +func (g *GenerateKeysInput) Validate() error { + if g.Topology == nil { + return errors.New("topology not set") + } + if len(g.Topology.DonsMetadata) == 0 { + return errors.New("metadata not set") + } + if g.Topology.WorkflowDONID == 0 { + return errors.New("workflow don id not set") + } + return nil +} + +type DonsToEVMKeys = map[uint32]*types.EVMKeys +type DonsToP2PKeys = map[uint32]*types.P2PKeys + +type GenerateKeysOutput struct { + EVMKeys DonsToEVMKeys + P2PKeys DonsToP2PKeys +} + +type GenerateSecretsInput struct { + DonMetadata *DonMetadata + EVMKeys *types.EVMKeys + P2PKeys *types.P2PKeys +} + +func (g *GenerateSecretsInput) Validate() error { + if g.DonMetadata == nil { + return errors.New("don metadata not set") + } + if g.EVMKeys != nil { + if len(g.EVMKeys.ChainIDs) == 0 { + return errors.New("chain ids not set") + } + if len(g.EVMKeys.EncryptedJSONs) == 0 { + return errors.New("encrypted jsons not set") + } + } + if g.P2PKeys != nil { + if len(g.P2PKeys.EncryptedJSONs) == 0 { + return errors.New("encrypted jsons not set") + } + } + + return nil +} + +type FullCLDEnvironmentInput struct { + JdOutput *jd.Output + BlockchainOutput *blockchain.Output + SethClient *seth.Client + NodeSetOutput []*WrappedNodeOutput + ExistingAddresses deployment.AddressBook + Topology *Topology +} + +func (f *FullCLDEnvironmentInput) Validate() error { + if f.JdOutput == nil { + return errors.New("jd output not set") + } + if f.BlockchainOutput == nil { + return errors.New("blockchain output not set") + } + if f.SethClient == nil { + return errors.New("seth client not set") + } + if len(f.NodeSetOutput) == 0 { + return errors.New("node set output not set") + } + if f.Topology == nil { + return errors.New("topology not set") + } + if len(f.Topology.DonsMetadata) == 0 { + return errors.New("metadata not set") + } + if f.Topology.WorkflowDONID == 0 { + return errors.New("workflow don id not set") + } + return nil +} + +type FullCLDEnvironmentOutput struct { + Environment *deployment.Environment + DonTopology *DonTopology +} diff --git a/system-tests/lib/crypto/evm.go b/system-tests/lib/crypto/evm.go new file mode 100644 index 00000000000..597677df17c --- /dev/null +++ b/system-tests/lib/crypto/evm.go @@ -0,0 +1,21 @@ +package crypto + +import ( + "github.com/smartcontractkit/chainlink-testing-framework/framework/clclient" + "github.com/smartcontractkit/chainlink/system-tests/lib/types" +) + +func GenerateEVMKeys(password string, n int) (*types.EVMKeys, error) { + result := &types.EVMKeys{ + Password: password, + } + for i := 0; i < n; i++ { + key, addr, err := clclient.NewETHKey(password) + if err != nil { + return result, nil + } + result.EncryptedJSONs = append(result.EncryptedJSONs, key) + result.PublicAddresses = append(result.PublicAddresses, addr) + } + return result, nil +} diff --git a/system-tests/lib/crypto/p2p.go b/system-tests/lib/crypto/p2p.go new file mode 100644 index 00000000000..83e77cb472b --- /dev/null +++ b/system-tests/lib/crypto/p2p.go @@ -0,0 +1,31 @@ +package crypto + +import ( + "encoding/hex" + + "github.com/smartcontractkit/chainlink/system-tests/lib/types" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" + "github.com/smartcontractkit/chainlink/v2/core/utils" +) + +func GenerateP2PKeys(password string, n int) (*types.P2PKeys, error) { + result := &types.P2PKeys{ + Password: password, + } + for i := 0; i < n; i++ { + key, err := p2pkey.NewV2() + if err != nil { + return nil, err + } + d, err := key.ToEncryptedJSON(password, utils.DefaultScryptParams) + if err != nil { + return nil, err + } + + result.EncryptedJSONs = append(result.EncryptedJSONs, d) + result.PeerIDs = append(result.PeerIDs, key.PeerID().String()) + result.PublicHexKeys = append(result.PublicHexKeys, key.PublicKeyHex()) + result.PrivateKeys = append(result.PrivateKeys, hex.EncodeToString(key.Raw())) + } + return result, nil +} diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 458d8fd2054..f805675e805 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -15,6 +15,7 @@ replace github.com/smartcontractkit/chainlink/deployment => ../../deployment require ( github.com/ethereum/go-ethereum v1.15.0 github.com/google/uuid v1.6.0 + github.com/pelletier/go-toml/v2 v2.2.3 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.33.0 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250221174903-e1e47fdb11b0 @@ -301,7 +302,6 @@ require ( github.com/otiai10/copy v1.14.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect diff --git a/system-tests/lib/types/crypto.go b/system-tests/lib/types/crypto.go new file mode 100644 index 00000000000..3decc24c07c --- /dev/null +++ b/system-tests/lib/types/crypto.go @@ -0,0 +1,18 @@ +package types + +import "github.com/ethereum/go-ethereum/common" + +type EVMKeys struct { + EncryptedJSONs [][]byte + PublicAddresses []common.Address + Password string + ChainIDs []int +} + +type P2PKeys struct { + EncryptedJSONs [][]byte + PeerIDs []string + PublicHexKeys []string + PrivateKeys []string + Password string +} diff --git a/system-tests/lib/types/node.go b/system-tests/lib/types/node.go new file mode 100644 index 00000000000..59ea5049f92 --- /dev/null +++ b/system-tests/lib/types/node.go @@ -0,0 +1,23 @@ +package types + +type NodeEthKey struct { + JSON string `toml:"JSON"` + Password string `toml:"Password"` + ChainID int `toml:"ID"` +} + +type NodeP2PKey struct { + JSON string `toml:"JSON"` + Password string `toml:"Password"` +} + +type NodeEthKeyWrapper struct { + EthKeys []NodeEthKey `toml:"Keys"` +} + +type NodeSecret struct { + EthKeys NodeEthKeyWrapper `toml:"EVM"` + P2PKey NodeP2PKey `toml:"P2PKey"` + // Add more fields as needed to reflect 'Secrets' struct from /core/config/toml/types.go + // We can't use the original struct, because it's using custom types that serlialize secrets to 'xxxxx' +} diff --git a/system-tests/tests/smoke/capabilities/environment-ci.toml b/system-tests/tests/smoke/capabilities/environment-one-don-ci.toml similarity index 100% rename from system-tests/tests/smoke/capabilities/environment-ci.toml rename to system-tests/tests/smoke/capabilities/environment-one-don-ci.toml diff --git a/system-tests/tests/smoke/capabilities/environment.toml b/system-tests/tests/smoke/capabilities/environment-one-don.toml similarity index 78% rename from system-tests/tests/smoke/capabilities/environment.toml rename to system-tests/tests/smoke/capabilities/environment-one-don.toml index a114520324f..cbe143bde27 100644 --- a/system-tests/tests/smoke/capabilities/environment.toml +++ b/system-tests/tests/smoke/capabilities/environment-one-don.toml @@ -16,8 +16,9 @@ feed_id = "018bfe8840700040000000000000000000000000000000000000000000000000" use_cre_cli = true - should_compile_new_workflow = false + should_compile_new_workflow = true # workflow_folder_location = "path-to-folder-with-main.go-of-your-workflow" + workflow_folder_location = "/Users/bartektofel/Downloads/workflow_test" [workflow_config.dependencies] capabilities_version = "v1.0.0-alpha" @@ -41,9 +42,9 @@ [[nodesets.node_specs]] [nodesets.node_specs.node] - # docker_ctx = "../../../.." - # docker_file = "plugins/chainlink.Dockerfile" - image = "chainlink-tmp" + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" user_config_overrides = """ [Feature] LogPoller = true @@ -60,9 +61,9 @@ [[nodesets.node_specs]] [nodesets.node_specs.node] - # docker_ctx = "../../../.." - # docker_file = "plugins/chainlink.Dockerfile" - image = "chainlink-tmp" + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -80,9 +81,9 @@ [[nodesets.node_specs]] [nodesets.node_specs.node] - # docker_ctx = "../../../.." - # docker_file = "plugins/chainlink.Dockerfile" - image = "chainlink-tmp" + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -100,9 +101,9 @@ [[nodesets.node_specs]] [nodesets.node_specs.node] - # docker_ctx = "../../../.." - # docker_file = "plugins/chainlink.Dockerfile" - image = "chainlink-tmp" + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -120,9 +121,9 @@ [[nodesets.node_specs]] [nodesets.node_specs.node] - # docker_ctx = "../../../.." - # docker_file = "plugins/chainlink.Dockerfile" - image = "chainlink-tmp" + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] diff --git a/system-tests/tests/smoke/capabilities/environment-multi-don-ci.toml b/system-tests/tests/smoke/capabilities/environment-three-dons-ci.toml similarity index 84% rename from system-tests/tests/smoke/capabilities/environment-multi-don-ci.toml rename to system-tests/tests/smoke/capabilities/environment-three-dons-ci.toml index a5962707ad6..1af4d66db82 100644 --- a/system-tests/tests/smoke/capabilities/environment-multi-don-ci.toml +++ b/system-tests/tests/smoke/capabilities/environment-three-dons-ci.toml @@ -19,7 +19,7 @@ [workflow_config.compiled_config] binary_url = "https://gist.githubusercontent.com/Tofel/73d703157bafe65ab51f7e619c589091/raw/cb7b2a56b37e333fe0bdce07b79538c4ce332f5f/binary.wasm.br" - config_url = "https://gist.githubusercontent.com/Tofel/a261990c5b177fe58f304a52d0998e51/raw/2e28ee10feacf4e451a38fdfbdff8a38cf2628d8/config.json2891974493" + config_url = "https://gist.githubusercontent.com/Tofel/3635532a08bccd0adae54794e449f810/raw/b3cf08909153fe469b063f9870ac40ad2a70d768/three-dons-config.json_2025_02_26" [[nodesets]] nodes = 5 @@ -166,6 +166,33 @@ ListenAddresses = ['0.0.0.0:5001'] """ + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + +[[nodesets]] + nodes = 1 + override_mode = "each" + http_port_range_start = 10300 + name = "gateway" + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + [[nodesets.node_specs]] [nodesets.node_specs.node] diff --git a/system-tests/tests/smoke/capabilities/environment-multi-don.toml b/system-tests/tests/smoke/capabilities/environment-three-dons.toml similarity index 81% rename from system-tests/tests/smoke/capabilities/environment-multi-don.toml rename to system-tests/tests/smoke/capabilities/environment-three-dons.toml index 631edf5be94..5dd22fba9a9 100644 --- a/system-tests/tests/smoke/capabilities/environment-multi-don.toml +++ b/system-tests/tests/smoke/capabilities/environment-three-dons.toml @@ -12,8 +12,9 @@ feed_id = "018bfe8840700040000000000000000000000000000000000000000000000000" use_cre_cli = true - should_compile_new_workflow = false + should_compile_new_workflow = true # workflow_folder_location = "path-to-folder-with-main.go-of-your-workflow" + workflow_folder_location = "/Users/bartektofel/Downloads/workflow_test" [workflow_config.dependencies] capabilities_version = "v1.0.0-alpha" @@ -32,13 +33,13 @@ [nodesets.db] image = "postgres:12.0" port = 13000 - volume_name = "workflow_volume" [[nodesets.node_specs]] [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" user_config_overrides = """ [Feature] LogPoller = true @@ -57,6 +58,7 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -76,6 +78,7 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -95,6 +98,7 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -114,6 +118,7 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" capabilities = ["./amd64_cron"] user_config_overrides = """ [Feature] @@ -137,13 +142,13 @@ [nodesets.db] image = "postgres:12.0" port = 13100 - volume_name = "capabilities_volume" [[nodesets.node_specs]] [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" user_config_overrides = """ [Feature] LogPoller = true @@ -162,6 +167,7 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" user_config_overrides = """ [Feature] LogPoller = true @@ -180,6 +186,36 @@ [nodesets.node_specs.node] docker_ctx = "../../../.." docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + +[[nodesets]] + nodes = 1 + override_mode = "each" + http_port_range_start = 10300 + name = "gateway" + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" user_config_overrides = """ [Feature] LogPoller = true diff --git a/system-tests/tests/smoke/capabilities/environment-two-dons-ci.toml b/system-tests/tests/smoke/capabilities/environment-two-dons-ci.toml new file mode 100644 index 00000000000..a547b491c9e --- /dev/null +++ b/system-tests/tests/smoke/capabilities/environment-two-dons-ci.toml @@ -0,0 +1,155 @@ + +[blockchain_a] + type = "anvil" + docker_cmd_params = ["-b", "5"] + +[jd] + image = "injected-at-runtime" + +[fake] + port = 8171 + +[workflow_config] + workflow_name = "abcdefgasd" + feed_id = "018bfe8840700040000000000000000000000000000000000000000000000000" + + use_cre_cli = true + should_compile_new_workflow = false + workflow_folder_location = "not-needed-here" + + [workflow_config.dependencies] + capabilities_version = "v1.0.0-alpha" + cre_cli_version = "v0.0.2" + + [workflow_config.compiled_config] + binary_url = "https://gist.githubusercontent.com/Tofel/0a595b92554b375415925749cc126969/raw/cb7b2a56b37e333fe0bdce07b79538c4ce332f5f/binary.wasm.br" + # if fake is enabled AND we do not compile a new workflow, this config needs to use URL pointing to IP, on which Docker host is available in Linux systems + # since that's the OS of our CI runners. + config_url = "https://gist.githubusercontent.com/Tofel/2dfcfa19d8a2f486681e5b959527951b/raw/5c5503651a5f0fdd61726ca0a7e8b4462824e96e/two_dons_config.json_25_02_2025" + +[[nodesets]] + nodes = 5 + override_mode = "each" + http_port_range_start = 10100 + name = "workflow" + + [nodesets.db] + image = "postgres:12.0" + port = 13000 + volume_name = "workflow_volume" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + +[[nodesets]] + nodes = 1 + override_mode = "each" + http_port_range_start = 10300 + name = "gateway" + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + image = "injected-at-runtime" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ \ No newline at end of file diff --git a/system-tests/tests/smoke/capabilities/environment-two-dons.toml b/system-tests/tests/smoke/capabilities/environment-two-dons.toml new file mode 100644 index 00000000000..be86dd44590 --- /dev/null +++ b/system-tests/tests/smoke/capabilities/environment-two-dons.toml @@ -0,0 +1,168 @@ + +[blockchain_a] + type = "anvil" + docker_cmd_params = ["-b", "5"] + +[jd] + # change to your version + image = "jd-test-1:latest" + +# fake data provider used as a mocked price source +[fake] + port = 8171 + +[workflow_config] + workflow_name = "abcdefgasd" + feed_id = "018bfe8840700040000000000000000000000000000000000000000000000000" + + use_cre_cli = true + should_compile_new_workflow = true + # workflow_folder_location = "path-to-folder-with-main.go-of-your-workflow" + workflow_folder_location = "/Users/bartektofel/Downloads/workflow_test" + + [workflow_config.dependencies] + capabilities_version = "v1.0.0-alpha" + cre_cli_version = "v0.0.2" + + [workflow_config.compiled_config] + binary_url = "https://gist.githubusercontent.com/Tofel/73d703157bafe65ab51f7e619c589091/raw/cb7b2a56b37e333fe0bdce07b79538c4ce332f5f/binary.wasm.br" + config_url = "https://gist.githubusercontent.com/Tofel/a261990c5b177fe58f304a52d0998e51/raw/2e28ee10feacf4e451a38fdfbdff8a38cf2628d8/config.json2891974493" + +[[nodesets]] + nodes = 5 + override_mode = "each" + http_port_range_start = 10100 + name = "workflow" + + [nodesets.db] + image = "postgres:12.0" + port = 13000 + volume_name = "workflow_volume" + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + capabilities = ["./amd64_cron"] + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ + +[[nodesets]] + nodes = 1 + override_mode = "each" + http_port_range_start = 10300 + name = "gateway" + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + + [[nodesets.node_specs]] + + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "plugins/chainlink.Dockerfile" + # image = "chainlink-tmp" + user_config_overrides = """ + [Feature] + LogPoller = true + + [OCR2] + Enabled = true + DatabaseTimeout = '1s' + + [P2P.V2] + Enabled = true + ListenAddresses = ['0.0.0.0:5001'] + """ \ No newline at end of file diff --git a/system-tests/tests/smoke/capabilities/workflow_test.go b/system-tests/tests/smoke/capabilities/workflow_test.go index eeed08ff176..699aa1da928 100644 --- a/system-tests/tests/smoke/capabilities/workflow_test.go +++ b/system-tests/tests/smoke/capabilities/workflow_test.go @@ -6,6 +6,7 @@ import ( "net" "os" "runtime" + "strconv" "strings" "testing" "time" @@ -28,6 +29,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/seth" "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" cldlogger "github.com/smartcontractkit/chainlink/deployment/logger" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/feeds_consumer" "github.com/smartcontractkit/chainlink/v2/core/logger" @@ -41,6 +43,7 @@ import ( keystoneporconfig "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/config/por" libjobs "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs" keystonepor "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/jobs/por" + keystonesecrets "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/secrets" libenv "github.com/smartcontractkit/chainlink/system-tests/lib/cre/environment" keystonetypes "github.com/smartcontractkit/chainlink/system-tests/lib/cre/types" libcrecli "github.com/smartcontractkit/chainlink/system-tests/lib/crecli" @@ -305,17 +308,18 @@ func logTestInfo(l zerolog.Logger, feedID, workflowName, feedConsumerAddr, forwa l.Info().Msgf("KeystoneForwarder address: %s", forwarderAddr) } -func extraAllowedPortsAndIps(testLogger zerolog.Logger, fakePort int, nodeOutput *ns.Output) ([]string, []int, error) { +func extraAllowedPortsAndIps(testLogger zerolog.Logger, fakePort int, containerName string) ([]string, []int, error) { // we need to explicitly allow the port used by the fake data provider // and IP corresponding to host.docker.internal or the IP of the host machine, if we are running on Linux, // because that's where the fake data provider is running var hostIP string var err error + // TODO add handling for CRIB as none of the current cases will work in k8s system := runtime.GOOS switch system { case "darwin": - hostIP, err = libdon.ResolveHostDockerInternaIP(testLogger, nodeOutput) + hostIP, err = libdon.ResolveHostDockerInternaIP(testLogger, containerName) case "linux": // for linux framework already returns an IP, so we don't need to resolve it, // but we need to remove the http:// prefix @@ -352,14 +356,10 @@ type InfrastructureInput struct { type InfrastructureOutput struct { chainSelector uint64 - nodeOuput []*keystonetypes.WrappedNodeOutput blockchainOutput *blockchain.Output jdOutput *jd.Output - cldEnv *deployment.Environment - donTopology *keystonetypes.DonTopology sethClient *seth.Client deployerPrivateKey string - gatewayConnector *keystonetypes.GatewayConnectorOutput } func CreateInfrastructure( @@ -431,55 +431,12 @@ func CreateInfrastructure( } } - nodeOutput := make([]*keystonetypes.WrappedNodeOutput, 0, len(input.nodeSetInput)) - for _, nsInput := range input.nodeSetInput { - nodeset, nodesetErr := ns.NewSharedDBNodeSet(nsInput.Input, blockchainOutput) - if nodesetErr != nil { - return nil, errors.Wrapf(nodesetErr, "failed to deploy node set names %s", nsInput.Name) - } - - nodeOutput = append(nodeOutput, &keystonetypes.WrappedNodeOutput{ - Output: nodeset, - NodeSetName: nsInput.Name, - Capabilities: nsInput.Capabilities, - }) - } - - // Prepare the CLD environment and figure out DON topology; configure chains for nodes and job distributor - // Ugly glue hack ¯\_(ツ)_/¯ - cldEnv, donTopology, err := libenv.BuildTopologyAndCLDEnvironment(cldLogger, input.nodeSetInput, jdOutput, nodeOutput, blockchainOutput, sethClient) - if err != nil { - return nil, errors.Wrap(err, "failed to build topology and CLD environment") - } - - // Fund the nodes - for _, metaDon := range donTopology.MetaDons { - for _, node := range metaDon.DON.Nodes { - _, err := libfunding.SendFunds(zerolog.Logger{}, sethClient, libtypes.FundsToSend{ - ToAddress: common.HexToAddress(node.AccountAddr[sethClient.Cfg.Network.ChainID]), - Amount: big.NewInt(5000000000000000000), - PrivateKey: sethClient.MustGetRootPrivateKey(), - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to send funds to node %s", node.AccountAddr[sethClient.Cfg.Network.ChainID]) - } - } - } - return &InfrastructureOutput{ chainSelector: chainSelector, - nodeOuput: nodeOutput, blockchainOutput: blockchainOutput, jdOutput: jdOutput, - cldEnv: cldEnv, - donTopology: donTopology, sethClient: sethClient, deployerPrivateKey: pkey, - gatewayConnector: &keystonetypes.GatewayConnectorOutput{ - Path: "/node", - Port: 5003, - // do not set the host, it will be resolved automatically - }, }, nil } @@ -490,6 +447,7 @@ type setupOutput struct { sethClient *seth.Client blockchainOutput *blockchain.Output donTopology *keystonetypes.DonTopology + nodeOutput []*keystonetypes.WrappedNodeOutput } func setupTestEnvironment(t *testing.T, testLogger zerolog.Logger, in *TestConfig, priceProvider PriceProvider, binaryDownloadOutput binaryDownloadOutput, mustSetCapabilitiesFn func(input []*ns.Input) []*keystonetypes.CapabilitiesAwareNodeSet) *setupOutput { @@ -499,34 +457,176 @@ func setupTestEnvironment(t *testing.T, testLogger zerolog.Logger, in *TestConfi nodeSetInput: mustSetCapabilitiesFn(in.NodeSets), blockchainInput: in.BlockchainA, } - envOutput, err := CreateInfrastructure(cldlogger.NewSingleFileLogger(t), testLogger, envInput) + singeFileLogger := cldlogger.NewSingleFileLogger(t) + envOutput, err := CreateInfrastructure(singeFileLogger, testLogger, envInput) require.NoError(t, err, "failed to start environment") // Deploy keystone contracts (forwarder, capability registry, ocr3 capability, workflow registry) + // but first, we need to create deployment.Environment that will contain only chain information in order to deploy contracts with the CLD + chainsConfig := []devenv.ChainConfig{ + { + ChainID: envOutput.sethClient.Cfg.Network.ChainID, + ChainName: envOutput.sethClient.Cfg.Network.Name, + ChainType: strings.ToUpper(envOutput.blockchainOutput.Family), + WSRPCs: []devenv.CribRPCs{{ + External: envOutput.blockchainOutput.Nodes[0].HostWSUrl, + Internal: envOutput.blockchainOutput.Nodes[0].DockerInternalWSUrl, + }}, + HTTPRPCs: []devenv.CribRPCs{{ + External: envOutput.blockchainOutput.Nodes[0].HostHTTPUrl, + Internal: envOutput.blockchainOutput.Nodes[0].DockerInternalHTTPUrl, + }}, + DeployerKey: envOutput.sethClient.NewTXOpts(seth.WithNonce(nil)), // set nonce to nil, so that it will be fetched from the RPC node + }, + } + + chains, err := devenv.NewChains(singeFileLogger, chainsConfig) + require.NoError(t, err, "failed to create chains") + + chainsOnlyCld := &deployment.Environment{ + Logger: singeFileLogger, + Chains: chains, + ExistingAddresses: deployment.NewMemoryAddressBook(), + } + keystoneContractsInput := &keystonetypes.KeystoneContractsInput{ ChainSelector: envOutput.chainSelector, - CldEnv: envOutput.cldEnv, + CldEnv: chainsOnlyCld, } keystoneContractsOutput, err := libcontracts.DeployKeystone(testLogger, keystoneContractsInput) require.NoError(t, err, "failed to deploy keystone contracts") - // Configure Workflow Registry + // Translate node input to structure required further down the road and put as much information + // as we have at this point in labels. It will be used to generate node configs + topology, err := libdon.BuildTopology(envInput.nodeSetInput) + require.NoError(t, err, "failed to build input DON topology") + + // Generate EVM and P2P keys, which are needed to prepare the node configs + // That way we can pass them final configs and do away with restarting the nodes + var keys *keystonetypes.GenerateKeysOutput + chainIDInt, err := strconv.Atoi(envOutput.blockchainOutput.ChainID) + require.NoError(t, err, "failed to convert chain ID to int") + + generateKeysInput := &keystonetypes.GenerateKeysInput{ + GenerateEVMKeysForChainIDs: []int{chainIDInt}, + GenerateP2PKeys: true, + Topology: topology, + Password: "", // since the test runs on private ephemeral blockchain we don't use real keys and do not care a lot about the password + } + keys, err = libdon.GenereteKeys(generateKeysInput) + require.NoError(t, err, "failed to generate keys") + + topology, err = libdon.AddKeysToTopology(topology, keys) + require.NoError(t, err, "failed to add keys to topology") + + // Configure Workflow Registry contract workflowRegistryInput := &keystonetypes.WorkflowRegistryInput{ ChainSelector: envOutput.chainSelector, - CldEnv: envOutput.cldEnv, - AllowedDonIDs: []uint32{envOutput.donTopology.WorkflowDONID}, + CldEnv: chainsOnlyCld, + AllowedDonIDs: []uint32{topology.WorkflowDONID}, WorkflowOwners: []common.Address{envOutput.sethClient.MustGetRootKeyAddress()}, } _, err = libcontracts.ConfigureWorkflowRegistry(testLogger, workflowRegistryInput) require.NoError(t, err, "failed to configure workflow registry") + + // Allow extra IPs and ports for the fake data provider, which is running on host machine and requires explicit whitelisting + // If using live endpoint, we don't need to do this + var extraAllowedIPs []string + var extraAllowedPorts []int + if _, ok := priceProvider.(*FakePriceProvider); ok { + // it doesn't really matter which container we will use to resolve the host.docker.internal IP, it will be the same for all of them + // here we will blokchain container, because by that time it will be running + extraAllowedIPs, extraAllowedPorts, err = extraAllowedPortsAndIps(testLogger, in.Fake.Port, envOutput.blockchainOutput.ContainerName) + require.NoError(t, err, "failed to get extra allowed ports and IPs") + } + + peeringData, err := libdon.FindPeeringData(topology) + require.NoError(t, err, "failed to get peering data") + + for i, donMetadata := range topology.DonsMetadata { + config, configErr := keystoneporconfig.GenerateConfigs( + keystonetypes.GeneratePoRConfigsInput{ + DonMetadata: donMetadata, + BlockchainOutput: envOutput.blockchainOutput, + DonID: donMetadata.ID, + Flags: donMetadata.Flags, + PeeringData: peeringData, + CapabilitiesRegistryAddress: keystoneContractsOutput.CapabilitiesRegistryAddress, + WorkflowRegistryAddress: keystoneContractsOutput.WorkflowRegistryAddress, + ForwarderAddress: keystoneContractsOutput.ForwarderAddress, + GatewayConnectorOutput: topology.GatewayConnectorOutput, + }, + ) + require.NoError(t, configErr, "failed to define config for DON %d", donMetadata.ID) + + secretsInput := &keystonetypes.GenerateSecretsInput{ + DonMetadata: donMetadata, + } + + if evmKeys, ok := keys.EVMKeys[donMetadata.ID]; ok { + secretsInput.EVMKeys = evmKeys + } + + if p2pKeys, ok := keys.P2PKeys[donMetadata.ID]; ok { + secretsInput.P2PKeys = p2pKeys + } + + // EVM and P2P keys will be provided to nodes as secrets + secrets, secretsErr := keystonesecrets.GenerateSecrets( + secretsInput, + ) + require.NoError(t, secretsErr, "failed to define secrets for DON %d", donMetadata.ID) + + for j := range donMetadata.NodesMetadata { + envInput.nodeSetInput[i].NodeSpecs[j].Node.TestConfigOverrides = config[j] + envInput.nodeSetInput[i].NodeSpecs[j].Node.TestSecretsOverrides = secrets[j] + } + } + + nodeOutput := make([]*keystonetypes.WrappedNodeOutput, 0, len(envInput.nodeSetInput)) + for _, nodeSetInput := range envInput.nodeSetInput { + nodeset, nodesetErr := ns.NewSharedDBNodeSet(nodeSetInput.Input, envOutput.blockchainOutput) + require.NoError(t, nodesetErr, "failed to deploy node set named %s", nodeSetInput.Name) + + nodeOutput = append(nodeOutput, &keystonetypes.WrappedNodeOutput{ + Output: nodeset, + NodeSetName: nodeSetInput.Name, + Capabilities: nodeSetInput.Capabilities, + }) + } + + // Prepare the CLD environment that's required by the keystone changeset + // Ugly glue hack ¯\_(ツ)_/¯ + fullCldInput := &keystonetypes.FullCLDEnvironmentInput{ + JdOutput: envOutput.jdOutput, + BlockchainOutput: envOutput.blockchainOutput, + SethClient: envOutput.sethClient, + NodeSetOutput: nodeOutput, + ExistingAddresses: chainsOnlyCld.ExistingAddresses, + Topology: topology, + } + fullCldOutput, err := libenv.BuildFullCLDEnvironment(singeFileLogger, fullCldInput) + require.NoError(t, err, "failed to build chainlink deployment environment") + + // Fund the nodes + for _, metaDon := range fullCldOutput.DonTopology.DonsWithMetadata { + for _, node := range metaDon.DON.Nodes { + _, fundingErr := libfunding.SendFunds(zerolog.Logger{}, envOutput.sethClient, libtypes.FundsToSend{ + ToAddress: common.HexToAddress(node.AccountAddr[envOutput.sethClient.Cfg.Network.ChainID]), + Amount: big.NewInt(5000000000000000000), + PrivateKey: envOutput.sethClient.MustGetRootPrivateKey(), + }) + require.NoError(t, fundingErr, "failed to send funds to node %s", node.AccountAddr[envOutput.sethClient.Cfg.Network.ChainID]) + } + } // Universal setup -- END // Workflow-specific configuration -- START deployFeedConsumerInput := &keystonetypes.DeployFeedConsumerInput{ ChainSelector: envOutput.chainSelector, - CldEnv: envOutput.cldEnv, + CldEnv: chainsOnlyCld, } deployFeedsConsumerOutput, err := libcontracts.DeployFeedsConsumer(testLogger, deployFeedConsumerInput) require.NoError(t, err, "failed to deploy feeds consumer") @@ -544,7 +644,7 @@ func setupTestEnvironment(t *testing.T, testLogger zerolog.Logger, in *TestConfi registerInput := registerPoRWorkflowInput{ WorkflowConfig: in.WorkflowConfig, chainSelector: envOutput.chainSelector, - workflowDonID: envOutput.donTopology.WorkflowDONID, + workflowDonID: fullCldOutput.DonTopology.WorkflowDonID, feedID: in.WorkflowConfig.FeedID, workflowRegistryAddress: keystoneContractsOutput.WorkflowRegistryAddress, feedConsumerAddress: deployFeedsConsumerOutput.FeedConsumerAddress, @@ -558,58 +658,44 @@ func setupTestEnvironment(t *testing.T, testLogger zerolog.Logger, in *TestConfi err = registerPoRWorkflow(registerInput) require.NoError(t, err, "failed to register PoR workflow") - // Workflow-specific configuration -- END - // Universal setup -- CONTINUED - // Allow extra IPs and ports for the fake data provider, which is running on host machine and requires explicit whitelisting - var extraAllowedIPs []string - var extraAllowedPorts []int - if _, ok := priceProvider.(*FakePriceProvider); ok { - extraAllowedIPs, extraAllowedPorts, err = extraAllowedPortsAndIps(testLogger, in.Fake.Port, envOutput.donTopology.MetaDons[0].NodeOutput.Output) - require.NoError(t, err, "failed to get extra allowed ports and IPs") - } - - // Prepare job specs and node configs - configsAndJobsInput := jobsAndConfigsInput{ - donTopology: envOutput.donTopology, - blockchainOutput: envOutput.blockchainOutput, - gatewayConnectorOutput: envOutput.gatewayConnector, - workflowRegistryAddress: keystoneContractsOutput.WorkflowRegistryAddress, - forwarderAddress: keystoneContractsOutput.ForwarderAddress, - capabilitiesRegistryAddress: keystoneContractsOutput.CapabilitiesRegistryAddress, - ocr3capabilityAddress: keystoneContractsOutput.OCR3CapabilityAddress, - cldEnv: envOutput.cldEnv, - extraAllowedIPs: extraAllowedIPs, - extraAllowedPorts: extraAllowedPorts, + donToJobSpecs, jobSpecsErr := keystonepor.GenerateJobSpecs( + &keystonetypes.GeneratePoRJobSpecsInput{ + BlockchainOutput: envOutput.blockchainOutput, + DonsWithMetadata: fullCldOutput.DonTopology.DonsWithMetadata, + OCR3CapabilityAddress: keystoneContractsOutput.OCR3CapabilityAddress, + ExtraAllowedPorts: extraAllowedPorts, + ExtraAllowedIPs: extraAllowedIPs, + CronCapBinName: cronCapabilityAssetFile, + GatewayConnectorOutput: *topology.GatewayConnectorOutput, + }, + ) + require.NoError(t, jobSpecsErr, "failed to define job specs for DONs") + + // Create jobs + createJobsInput := keystonetypes.CreateJobsInput{ + CldEnv: fullCldOutput.Environment, + DonTopology: fullCldOutput.DonTopology, + DonToJobSpecs: donToJobSpecs, } - configsAndJobsOutput, err := prepareJobSpecsAndNodeConfigs(configsAndJobsInput) - require.NoError(t, err, "failed to prepare job specs and node configs") - - // Configure nodes and create jobs - configureDonInput := keystonetypes.ConfigureDonInput{ - CldEnv: envOutput.cldEnv, - BlockchainOutput: envOutput.blockchainOutput, - JdOutput: envOutput.jdOutput, - DonTopology: envOutput.donTopology, - DonToJobSpecs: configsAndJobsOutput.donToJobSpecs, - DonToConfigOverrides: configsAndJobsOutput.nodeToConfigOverrides, - } - _, err = libdon.Configure(t, testLogger, configureDonInput) + err = libdon.CreateJobs(testLogger, createJobsInput) require.NoError(t, err, "failed to configure nodes and create jobs") + // Workflow-specific configuration -- END + // Universal setup -- CONTINUED // CAUTION: It is crucial to configure OCR3 jobs on nodes before configuring the workflow contracts. // Wait for OCR listeners to be ready before setting the configuration. // If the ConfigSet event is missed, OCR protocol will not start. - testLogger.Info().Msg("Waiting 30s for OCR listeners to be ready...") - time.Sleep(30 * time.Second) - testLogger.Info().Msg("Proceeding to set OCR3 configuration.") + testLogger.Info().Msg("Waiting 45s for OCR listeners to be ready...") + time.Sleep(45 * time.Second) + testLogger.Info().Msg("Proceeding to set OCR3 and Keystone configuration...") // Configure the Forwarder, OCR3 and Capabilities contracts configureKeystoneInput := keystonetypes.ConfigureKeystoneInput{ ChainSelector: envOutput.chainSelector, - CldEnv: envOutput.cldEnv, - DonTopology: envOutput.donTopology, + CldEnv: fullCldOutput.Environment, + Topology: topology, } err = libcontracts.ConfigureKeystone(configureKeystoneInput) require.NoError(t, err, "failed to configure keystone contracts") @@ -625,94 +711,118 @@ func setupTestEnvironment(t *testing.T, testLogger zerolog.Logger, in *TestConfi forwarderAddress: keystoneContractsOutput.ForwarderAddress, sethClient: envOutput.sethClient, blockchainOutput: envOutput.blockchainOutput, - donTopology: envOutput.donTopology, + donTopology: fullCldOutput.DonTopology, + nodeOutput: nodeOutput, } } -type jobsAndConfigsInput struct { - donTopology *keystonetypes.DonTopology - blockchainOutput *blockchain.Output - gatewayConnectorOutput *keystonetypes.GatewayConnectorOutput - workflowRegistryAddress common.Address - forwarderAddress common.Address - capabilitiesRegistryAddress common.Address - ocr3capabilityAddress common.Address - cldEnv *deployment.Environment - extraAllowedIPs []string - extraAllowedPorts []int -} +func TestKeystoneWithOCR3Workflow_SingleDon_MockedPrice(t *testing.T) { + testLogger := framework.L -type jobsAndConfigsOutput struct { - donToJobSpecs keystonetypes.DonsToJobSpecs - nodeToConfigOverrides keystonetypes.DonsToConfigOverrides -} + // Load and validate test configuration + in, err := framework.Load[TestConfig](t) + require.NoError(t, err, "couldn't load test config") + validateEnvVars(t, in) + require.Len(t, in.NodeSets, 1, "expected 1 node set in the test config") -func prepareJobSpecsAndNodeConfigs(input jobsAndConfigsInput) (*jobsAndConfigsOutput, error) { - peeringData, err := libdon.FindPeeringData(input.donTopology.MetaDons) - if err != nil { - return nil, errors.Wrap(err, "failed to get peering data") - } + binaryDownloadOutput, err := downloadBinaryFiles(in) + require.NoError(t, err, "failed to download binary files") - // prepare node configs - donToConfigs := make(keystonetypes.DonsToConfigOverrides) - var configErr error - for _, donTopology := range input.donTopology.MetaDons { - donToConfigs[donTopology.ID], configErr = keystoneporconfig.GenerateConfigs( - keystonetypes.GeneratePoRConfigsInput{ - Don: donTopology.DON, - NodeInput: donTopology.NodeInput, - BlockchainOutput: input.blockchainOutput, - DonID: donTopology.ID, - Flags: donTopology.Flags, - PeeringData: peeringData, - CapabilitiesRegistryAddress: input.capabilitiesRegistryAddress, - WorkflowRegistryAddress: input.workflowRegistryAddress, - ForwarderAddress: input.forwarderAddress, - GatewayConnectorOutput: input.gatewayConnectorOutput, + // Assign all capabilities to the single node set + mustSetCapabilitiesFn := func(input []*ns.Input) []*keystonetypes.CapabilitiesAwareNodeSet { + return []*keystonetypes.CapabilitiesAwareNodeSet{ + { + Input: input[0], + Capabilities: keystonetypes.SingleDonFlags, + DONTypes: []string{keystonetypes.WorkflowDON, keystonetypes.GatewayDON}, + BootstrapNodeIndex: 0, + GatewayNodeIndex: 0, }, - ) - if configErr != nil { - return nil, errors.Wrapf(configErr, "failed to define config for DON %d", donTopology.ID) } } - // define jobs - donToJobSpecs := make(map[uint32]keystonetypes.DonJobs) - var jobSpecsErr error - for _, donTopology := range input.donTopology.MetaDons { - donToJobSpecs[donTopology.ID], jobSpecsErr = keystonepor.GenerateJobSpecs( - keystonetypes.GeneratePoRJobSpecsInput{ - CldEnv: input.cldEnv, - Don: donTopology.DON, - NodeOutput: donTopology.NodeOutput, - BlockchainOutput: input.blockchainOutput, - DonID: donTopology.ID, - Flags: donTopology.Flags, - OCR3CapabilityAddress: input.ocr3capabilityAddress, - ExtraAllowedPorts: input.extraAllowedPorts, - ExtraAllowedIPs: input.extraAllowedIPs, - CronCapBinName: cronCapabilityAssetFile, - GatewayConnectorOutput: *input.gatewayConnectorOutput, - }, + priceProvider, priceErr := NewFakePriceProvider(testLogger, in.Fake) + require.NoError(t, priceErr, "failed to create fake price provider") + + setupOutput := setupTestEnvironment(t, testLogger, in, priceProvider, *binaryDownloadOutput, mustSetCapabilitiesFn) + + // Log extra information that might help debugging + t.Cleanup(func() { + if t.Failed() { + logTestInfo(testLogger, in.WorkflowConfig.FeedID, in.WorkflowConfig.WorkflowName, setupOutput.feedsConsumerAddress.Hex(), setupOutput.forwarderAddress.Hex()) + + logDir := fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, t.Name()) + + removeErr := os.RemoveAll(logDir) + if removeErr != nil { + testLogger.Error().Err(removeErr).Msg("failed to remove log directory") + return + } + + _, saveErr := framework.SaveContainerLogs(logDir) + if saveErr != nil { + testLogger.Error().Err(saveErr).Msg("failed to save container logs") + return + } + + debugDons := make([]*keystonetypes.DebugDon, 0, len(setupOutput.donTopology.DonsWithMetadata)) + for i, donWithMetadata := range setupOutput.donTopology.DonsWithMetadata { + containerNames := make([]string, 0, len(donWithMetadata.NodesMetadata)) + for _, output := range setupOutput.nodeOutput[i].Output.CLNodes { + containerNames = append(containerNames, output.Node.ContainerName) + } + debugDons = append(debugDons, &keystonetypes.DebugDon{ + NodesMetadata: donWithMetadata.NodesMetadata, + Flags: donWithMetadata.Flags, + ContainerNames: containerNames, + }) + } + + debugInput := keystonetypes.DebugInput{ + DebugDons: debugDons, + BlockchainOutput: setupOutput.blockchainOutput, + } + lidebug.PrintTestDebug(t.Name(), testLogger, debugInput) + } + }) + + testLogger.Info().Msg("Waiting for feed to update...") + timeout := 5 * time.Minute // It can take a while before the first report is produced, particularly on CI. + + feedsConsumerInstance, err := feeds_consumer.NewKeystoneFeedsConsumer(setupOutput.feedsConsumerAddress, setupOutput.sethClient.Client) + require.NoError(t, err, "failed to create feeds consumer instance") + + startTime := time.Now() + feedBytes := common.HexToHash(in.WorkflowConfig.FeedID) + + assert.Eventually(t, func() bool { + elapsed := time.Since(startTime).Round(time.Second) + price, _, err := feedsConsumerInstance.GetPrice( + setupOutput.sethClient.NewCallOpts(), + feedBytes, ) - if jobSpecsErr != nil { - return nil, errors.Wrapf(jobSpecsErr, "failed to define job specs for DON %d", donTopology.ID) + require.NoError(t, err, "failed to get price from Keystone Consumer contract") + + hasNextPrice := setupOutput.priceProvider.NextPrice(price, elapsed) + if !hasNextPrice { + testLogger.Info().Msgf("Feed not updated yet, waiting for %s", elapsed) } - } - return &jobsAndConfigsOutput{ - nodeToConfigOverrides: donToConfigs, - donToJobSpecs: donToJobSpecs, - }, nil + return !hasNextPrice + }, timeout, 10*time.Second, "feed did not update, timeout after: %s", timeout) + + require.EqualValues(t, priceProvider.ExpectedPrices(), priceProvider.ActualPrices(), "prices do not match") + testLogger.Info().Msgf("All %d prices were found in the feed", len(priceProvider.ExpectedPrices())) } -func TestKeystoneWithOCR3Workflow_SingleDon_MockedPrice(t *testing.T) { + +func TestKeystoneWithOCR3Workflow_GatewayDon_MockedPrice(t *testing.T) { testLogger := framework.L // Load and validate test configuration in, err := framework.Load[TestConfig](t) require.NoError(t, err, "couldn't load test config") validateEnvVars(t, in) - require.Len(t, in.NodeSets, 1, "expected 1 node set in the test config") + require.Len(t, in.NodeSets, 2, "expected 2 node sets in the test config") binaryDownloadOutput, err := downloadBinaryFiles(in) require.NoError(t, err, "failed to download binary files") @@ -721,9 +831,17 @@ func TestKeystoneWithOCR3Workflow_SingleDon_MockedPrice(t *testing.T) { mustSetCapabilitiesFn := func(input []*ns.Input) []*keystonetypes.CapabilitiesAwareNodeSet { return []*keystonetypes.CapabilitiesAwareNodeSet{ { - Input: input[0], - Capabilities: keystonetypes.SingleDonFlags, - DONType: keystonetypes.WorkflowDON, + Input: input[0], + Capabilities: keystonetypes.SingleDonFlags, + DONTypes: []string{keystonetypes.WorkflowDON}, + BootstrapNodeIndex: 0, + }, + { + Input: input[1], + Capabilities: []string{}, + DONTypes: []string{keystonetypes.GatewayDON}, // <----- it's crucial to set the correct DON type + GatewayNodeIndex: 0, + BootstrapNodeIndex: -1, // <----- it's crucial to indicate there's no bootstrap node }, } } @@ -752,8 +870,21 @@ func TestKeystoneWithOCR3Workflow_SingleDon_MockedPrice(t *testing.T) { return } + debugDons := make([]*keystonetypes.DebugDon, 0, len(setupOutput.donTopology.DonsWithMetadata)) + for i, donWithMetadata := range setupOutput.donTopology.DonsWithMetadata { + containerNames := make([]string, 0, len(donWithMetadata.NodesMetadata)) + for _, output := range setupOutput.nodeOutput[i].Output.CLNodes { + containerNames = append(containerNames, output.Node.ContainerName) + } + debugDons = append(debugDons, &keystonetypes.DebugDon{ + NodesMetadata: donWithMetadata.NodesMetadata, + Flags: donWithMetadata.Flags, + ContainerNames: containerNames, + }) + } + debugInput := keystonetypes.DebugInput{ - DonTopology: setupOutput.donTopology, + DebugDons: debugDons, BlockchainOutput: setupOutput.blockchainOutput, } lidebug.PrintTestDebug(t.Name(), testLogger, debugInput) @@ -785,18 +916,18 @@ func TestKeystoneWithOCR3Workflow_SingleDon_MockedPrice(t *testing.T) { return !hasNextPrice }, timeout, 10*time.Second, "feed did not update, timeout after: %s", timeout) - require.EqualValues(t, priceProvider.ExpectedPrices(), priceProvider.ActualPrices(), "prices do not match") + require.EqualValues(t, priceProvider.ExpectedPrices(), priceProvider.ActualPrices(), "pricesup do not match") testLogger.Info().Msgf("All %d prices were found in the feed", len(priceProvider.ExpectedPrices())) } -func TestKeystoneWithOCR3Workflow_TwoDons_LivePrice(t *testing.T) { +func TestKeystoneWithOCR3Workflow_ThreeDons_LivePrice(t *testing.T) { testLogger := framework.L // Load and validate test configuration in, err := framework.Load[TestConfig](t) require.NoError(t, err, "couldn't load test config") validateEnvVars(t, in) - require.Len(t, in.NodeSets, 2, "expected 2 node sets in the test config") + require.Len(t, in.NodeSets, 3, "expected 3 node sets in the test config") binaryDownloadOutput, err := downloadBinaryFiles(in) require.NoError(t, err, "failed to download binary files") @@ -804,14 +935,23 @@ func TestKeystoneWithOCR3Workflow_TwoDons_LivePrice(t *testing.T) { mustSetCapabilitiesFn := func(input []*ns.Input) []*keystonetypes.CapabilitiesAwareNodeSet { return []*keystonetypes.CapabilitiesAwareNodeSet{ { - Input: input[0], - Capabilities: []string{keystonetypes.OCR3Capability, keystonetypes.CustomComputeCapability, keystonetypes.CronCapability}, - DONType: keystonetypes.WorkflowDON, + Input: input[0], + Capabilities: []string{keystonetypes.OCR3Capability, keystonetypes.CustomComputeCapability, keystonetypes.CronCapability}, + DONTypes: []string{keystonetypes.WorkflowDON}, + BootstrapNodeIndex: 0, + }, + { + Input: input[1], + Capabilities: []string{keystonetypes.WriteEVMCapability}, + DONTypes: []string{keystonetypes.CapabilitiesDON}, // <----- it's crucial to set the correct DON type + BootstrapNodeIndex: 0, }, { - Input: input[1], - Capabilities: []string{keystonetypes.WriteEVMCapability}, - DONType: keystonetypes.CapabilitiesDON, // <----- it's crucial to set the correct DON type + Input: input[2], + Capabilities: []string{}, + DONTypes: []string{keystonetypes.GatewayDON}, // <----- it's crucial to set the correct DON type + GatewayNodeIndex: 0, + BootstrapNodeIndex: -1, // <----- it's crucial to indicate there's no bootstrap node }, } } @@ -838,8 +978,21 @@ func TestKeystoneWithOCR3Workflow_TwoDons_LivePrice(t *testing.T) { return } + debugDons := make([]*keystonetypes.DebugDon, 0, len(setupOutput.donTopology.DonsWithMetadata)) + for i, donWithMetadata := range setupOutput.donTopology.DonsWithMetadata { + containerNames := make([]string, 0, len(donWithMetadata.NodesMetadata)) + for _, output := range setupOutput.nodeOutput[i].Output.CLNodes { + containerNames = append(containerNames, output.Node.ContainerName) + } + debugDons = append(debugDons, &keystonetypes.DebugDon{ + NodesMetadata: donWithMetadata.NodesMetadata, + Flags: donWithMetadata.Flags, + ContainerNames: containerNames, + }) + } + debugInput := keystonetypes.DebugInput{ - DonTopology: setupOutput.donTopology, + DebugDons: debugDons, BlockchainOutput: setupOutput.blockchainOutput, } lidebug.PrintTestDebug(t.Name(), testLogger, debugInput)