Skip to content

Commit

Permalink
[TT-2001] keystone e2e tests - generate config first (#16565)
Browse files Browse the repository at this point in the history 
* modular keystone smoke test -> new module

* fix go.md, dynamically resolve gist ip, update forwarder address in CI config

* decouple lib from testing.T

* fix lints and CI test config

* go mod tidy + check returned err

* go mod tidy

* add single point of entry to configure keystone env

* safer keystone env field access

* make code more explicit

* separate test for single and multi DON

* remove conditional logic from test config

* fix lints

* use unique job names

* check only one price in the CI

* fix e2e test setup, allow using cached contract addresses and config

* remove one validator

* use tagged version of CTF

* go mod tidy

* fix keytsone test path

* CR changes: less public variables, labels for node roles, clearer function naming, more input structs

* add Validate() to all input structs

* go mod tidy

* try using latest version of github.com/smartcontractkit/ccip-owner-contracts

* add changeset

* update keystone e2e test name in gh config

* go mod tidy, update ci env config

* try to pregenerate all configs

* move price provider to dedicated file

* define config before starting the node

* update config

* remove commented out code

* wip: support single eth key secret

* working ETH keys import

* add back node.go

* import p2p key

* WIP#2

* support multiple eth keys

* working version

* fix eth keys plumbing and add test

* working verion after some consolidation

* rename vars

* working locally

* cleanup

* more clean up

* working version

* support gateway DON

* clean up, dedicated function to find label value, one more test (also added to CI)

* update deps

* use JD 0.9.0 in gateway DON test

* code review changes

* clearer expressions in TOML config

* add 3 dons keystone e2e test to CI

* do not use JD's proto labels

---------

Co-authored-by: krehermann <[email protected]>
  • Loading branch information
@Tofel @krehermann
Tofel and krehermann authored Feb 27, 2025
1 parent 2245fc4 commit 5cfb0e8
Show file tree
Hide file tree
Showing 29 changed files with 1,735 additions and 603 deletions.
26 changes: 22 additions & 4 deletions .github/e2e-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,25 +197,43 @@ runner-test-matrix:
E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version
GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos
CI: "true"
CTF_CONFIGS: "environment-ci.toml"
CTF_CONFIGS: "environment-one-don-ci.toml"
# Anvil developer key, not a secret
PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

- id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_TwoDons
- id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_GatewayDon
path: system-tests/tests/smoke/capabilities/workflow_test.go
test_env_type: docker
runs_on: ubuntu-latest
triggers:
- PR Workflow Engine E2E Core Tests
- Nightly E2E Tests
test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_TwoDons_LivePrice)$" -timeout 30m -count=1 -test.parallel=1 -json
test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_GatewayDon_MockedPrice)$" -timeout 30m -count=1 -test.parallel=1 -json
pyroscope_env: ci-smoke-capabilities-evm-simulated
test_env_vars:
E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins
E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version
GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos
CI: "true"
CTF_CONFIGS: "environment-multi-don-ci.toml"
CTF_CONFIGS: "environment-two-dons-ci.toml"
# Anvil developer key, not a secret
PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

- id: system-tests/smoke/capabilities/workflow_test.go:TestKeystoneWithOCR3Workflow_ThreeDons
path: system-tests/tests/smoke/capabilities/workflow_test.go
test_env_type: docker
runs_on: ubuntu-latest
triggers:
- PR Workflow Engine E2E Core Tests
- Nightly E2E Tests
test_cmd: cd system-tests/tests && go test github.com/smartcontractkit/chainlink/system-tests/tests/smoke/capabilities -v -run "^(TestKeystoneWithOCR3Workflow_ThreeDons_LivePrice)$" -timeout 30m -count=1 -test.parallel=1 -json
pyroscope_env: ci-smoke-capabilities-evm-simulated
test_env_vars:
E2E_TEST_CHAINLINK_VERSION: '{{ env.DEFAULT_CHAINLINK_PLUGINS_VERSION }}' # This is the chainlink version that has the plugins
E2E_JD_VERSION: 0.9.0 # there is no latest tag for this repo, so we need to specify the version
GITHUB_READ_TOKEN: '{{ env.GITHUB_API_TOKEN }}' # GATI-provided token that can read from capabilities and dev-platform repos
CI: "true"
CTF_CONFIGS: "environment-three-dons-ci.toml"
# Anvil developer key, not a secret
PRIVATE_KEY: "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"

Expand Down
1 change: 1 addition & 0 deletions deployment/environment/devenv/don.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChai
return fmt.Errorf("no OCR2 key bundle id found for node %s", n.Name)
}
n.Ocr2KeyBundleID = ocr2BundleId

// fetch node labels to know if the node is bootstrap or plugin
// if multi address is set, then it's a bootstrap node
isBootstrap := n.multiAddr != ""
Expand Down
55 changes: 38 additions & 17 deletions system-tests/lib/cre/contracts/contracts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags"
"github.com/smartcontractkit/chainlink/system-tests/lib/cre/types"

"github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node"
keystonenode "github.com/smartcontractkit/chainlink/system-tests/lib/cre/don/node"
)

Expand All @@ -30,13 +31,19 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {
return errors.Wrap(err, "input validation failed")
}

donCapabilities := make([]keystone_changeset.DonCapabilities, 0, len(input.DonTopology.MetaDons))
donCapabilities := make([]keystone_changeset.DonCapabilities, 0, len(input.Topology.DonsMetadata))

for _, donMetadata := range input.Topology.DonsMetadata {
// if it's only a gateway DON, we don't want to register it with the Capabilities Registry
// since it doesn't have any capabilities
if flags.HasOnlyOneFlag(donMetadata.Flags, types.GatewayDON) {
continue
}

for _, metaDon := range input.DonTopology.MetaDons {
var capabilities []keystone_changeset.DONCapabilityWithConfig

// check what capabilities each DON has and register them with Capabilities Registry contract
if flags.HasFlag(metaDon.Flags, types.CronCapability) {
if flags.HasFlag(donMetadata.Flags, types.CronCapability) {
capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{
Capability: kcr.CapabilitiesRegistryCapability{
LabelledName: "cron-trigger",
Expand All @@ -47,7 +54,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {
})
}

if flags.HasFlag(metaDon.Flags, types.CustomComputeCapability) {
if flags.HasFlag(donMetadata.Flags, types.CustomComputeCapability) {
capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{
Capability: kcr.CapabilitiesRegistryCapability{
LabelledName: "custom-compute",
Expand All @@ -58,7 +65,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {
})
}

if flags.HasFlag(metaDon.Flags, types.OCR3Capability) {
if flags.HasFlag(donMetadata.Flags, types.OCR3Capability) {
capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{
Capability: kcr.CapabilitiesRegistryCapability{
LabelledName: "offchain_reporting",
Expand All @@ -70,7 +77,7 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {
})
}

if flags.HasFlag(metaDon.Flags, types.WriteEVMCapability) {
if flags.HasFlag(donMetadata.Flags, types.WriteEVMCapability) {
capabilities = append(capabilities, keystone_changeset.DONCapabilityWithConfig{
Capability: kcr.CapabilitiesRegistryCapability{
LabelledName: "write_geth-testnet",
Expand All @@ -84,28 +91,33 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {

// Add support for new capabilities here as needed

donPeerIDs := make([]string, len(metaDon.DON.Nodes)-1)
for i, node := range metaDon.DON.Nodes {
if i == 0 {
continue
}
workerNodes, workerNodesErr := node.FindManyWithLabel(donMetadata.NodesMetadata, &types.Label{
Key: node.NodeTypeKey,
Value: types.WorkerNode,
}, node.EqualLabels)

if workerNodesErr != nil {
return errors.Wrap(workerNodesErr, "failed to find worker nodes")
}

donPeerIDs := make([]string, len(workerNodes))
for i, node := range workerNodes {
p2pID, err := keystonenode.ToP2PID(node, keystonenode.NoOpTransformFn)
if err != nil {
return errors.Wrapf(err, "failed to get p2p id for node %s", node.Name)
return errors.Wrapf(err, "failed to get p2p id for node %d", i)
}

donPeerIDs[i-1] = p2pID
donPeerIDs[i] = p2pID
}

// we only need to assign P2P IDs to NOPs, since `ConfigureInitialContractsChangeset` method
// will take care of creating DON to Nodes mapping
nop := keystone_changeset.NOP{
Name: fmt.Sprintf("NOP for %s DON", metaDon.NodeOutput.NodeSetName),
Name: fmt.Sprintf("NOP for %s DON", donMetadata.Name),
Nodes: donPeerIDs,
}

donName := metaDon.NodeOutput.NodeSetName + "-don"
donName := donMetadata.Name + "-don"
donCapabilities = append(donCapabilities, keystone_changeset.DonCapabilities{
Name: donName,
F: 1,
Expand All @@ -116,10 +128,19 @@ func ConfigureKeystone(input types.ConfigureKeystoneInput) error {

var transmissionSchedule []int

for _, metaDon := range input.DonTopology.MetaDons {
for _, metaDon := range input.Topology.DonsMetadata {
if flags.HasFlag(metaDon.Flags, types.OCR3Capability) {
workerNodes, workerNodesErr := node.FindManyWithLabel(metaDon.NodesMetadata, &types.Label{
Key: node.NodeTypeKey,
Value: types.WorkerNode,
}, node.EqualLabels)

if workerNodesErr != nil {
return errors.Wrap(workerNodesErr, "failed to find worker nodes")
}

// this schedule makes sure that all worker nodes are transmitting OCR3 reports
transmissionSchedule = []int{len(metaDon.DON.Nodes) - 1}
transmissionSchedule = []int{len(workerNodes)}
break
}
}
Expand Down
26 changes: 11 additions & 15 deletions system-tests/lib/cre/debug/debug.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/rs/zerolog"

ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
"github.com/smartcontractkit/chainlink-testing-framework/seth"

"github.com/smartcontractkit/chainlink/system-tests/lib/cre/flags"
Expand All @@ -40,8 +39,8 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) {
}
}()

for _, donTopology := range input.DonTopology.MetaDons {
logFiles, err := getLogFileHandles(testName, l, donTopology.NodeOutput.Output)
for _, debugDon := range input.DebugDons {
logFiles, err := getLogFileHandles(testName, l, debugDon)
if err != nil {
l.Error().Err(err).Msg("Failed to get log file handles. No debug information will be printed")
return
Expand All @@ -50,25 +49,25 @@ func PrintTestDebug(testName string, l zerolog.Logger, input types.DebugInput) {
allLogFiles = append(allLogFiles, logFiles...)

// assuming one bootstrap node
workflowNodeCount := len(donTopology.NodeOutput.CLNodes) - 1
workflowNodeCount := len(debugDon.ContainerNames) - 1

if flags.HasFlag(donTopology.Flags, types.WorkflowDON) {
if flags.HasFlag(debugDon.Flags, types.WorkflowDON) {
if !checkIfWorkflowWasExecuting(logFiles, workflowNodeCount) {
l.Error().Msg("❌ Workflow was not executing")
return
}
l.Info().Msg("✅ Workflow was executing")
}

if flags.HasFlag(donTopology.Flags, types.OCR3Capability) {
if flags.HasFlag(debugDon.Flags, types.OCR3Capability) {
if !checkIfOCRWasExecuting(logFiles, workflowNodeCount) {
l.Error().Msg("❌ OCR was not executing")
return
}
l.Info().Msg("✅ OCR was executing")
}

if flags.HasFlag(donTopology.Flags, types.WriteEVMCapability) {
if flags.HasFlag(debugDon.Flags, types.WriteEVMCapability) {
if !checkIfAtLeastOneReportWasSent(logFiles, workflowNodeCount) {
l.Error().Msg("❌ Reports were not sent")
return
Expand Down Expand Up @@ -167,21 +166,18 @@ func ReportTransmissions(logFiles []*os.File, l zerolog.Logger, wsRPCURL string)
}
}

func getLogFileHandles(testName string, l zerolog.Logger, ns *ns.Output) ([]*os.File, error) {
func getLogFileHandles(testName string, l zerolog.Logger, debugDon *types.DebugDon) ([]*os.File, error) {
var logFiles []*os.File

var belongsToCurrentEnv = func(filePath string) bool {
for i, clNode := range ns.CLNodes {
if clNode == nil {
continue
}

for i, containerName := range debugDon.ContainerNames {
// TODO check corresponding flag when looking for bootstrap node
// skip the first node, as it's the bootstrap node
if i == 0 {
continue
}

if strings.EqualFold(filePath, clNode.Node.ContainerName+".log") {
if strings.EqualFold(filePath, containerName+".log") {
return true
}
}
Expand All @@ -204,7 +200,7 @@ func getLogFileHandles(testName string, l zerolog.Logger, ns *ns.Output) ([]*os.
return nil
})

expectedLogCount := len(ns.CLNodes) - 1
expectedLogCount := len(debugDon.ContainerNames) - 1
if len(logFiles) != expectedLogCount {
l.Warn().Int("Expected", expectedLogCount).Int("Got", len(logFiles)).Msg("Number of log files does not match number of worker nodes. Some logs might be missing.")
}
Expand Down
5 changes: 3 additions & 2 deletions system-tests/lib/cre/don/config/definitions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func BootstrapEVM(donBootstrapNodePeerID string, chainID uint64, capabilitiesReg
)
}

func BoostrapDon2DonPeering(peeringData types.PeeringData) string {
func BoostrapDon2DonPeering(peeringData types.CapabilitiesPeeringData) string {
return fmt.Sprintf(`
[Capabilities.Peering.V2]
Enabled = true
Expand All @@ -71,7 +71,7 @@ func BoostrapDon2DonPeering(peeringData types.PeeringData) string {
//
// so that we are future-proof (for bootstrap too!)
// we'd need to have capabilitiesRegistryChainID too
func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData types.PeeringData, chainID uint64, capabilitiesRegistryAddress common.Address, httpRPC, wsRPC string) string {
func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData types.CapabilitiesPeeringData, chainID uint64, capabilitiesRegistryAddress common.Address, httpRPC, wsRPC string) string {
return fmt.Sprintf(`
[Feature]
LogPoller = true
Expand All @@ -93,6 +93,7 @@ func WorkerEVM(donBootstrapNodePeerID, donBootstrapNodeHost string, peeringData
[[EVM]]
ChainID = '%d'
AutoCreateKey = false
[[EVM.Nodes]]
Name = 'anvil'
Expand Down
Loading

0 comments on commit 5cfb0e8

Please sign in to comment.