diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index eba13cab3e5..4e0198a142f 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -28,6 +28,7 @@ import ( stateSyncCommands "github.com/onflow/flow-go/admin/commands/state_synchronization" storageCommands "github.com/onflow/flow-go/admin/commands/storage" "github.com/onflow/flow-go/cmd" + "github.com/onflow/flow-go/cmd/build" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -52,6 +53,7 @@ import ( followereng "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/requester" synceng "github.com/onflow/flow-go/engine/common/synchronization" + "github.com/onflow/flow-go/engine/common/version" "github.com/onflow/flow-go/engine/execution/computation/query" "github.com/onflow/flow-go/fvm/storage/derived" "github.com/onflow/flow-go/ledger" @@ -164,6 +166,7 @@ type AccessNodeConfig struct { registerCacheSize uint programCacheSize uint checkPayerBalance bool + versionControlEnabled bool } type PublicNetworkConfig struct { @@ -264,6 +267,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { registerCacheSize: 0, programCacheSize: 0, checkPayerBalance: false, + versionControlEnabled: true, } } @@ -311,6 +315,7 @@ type FlowAccessNodeBuilder struct { ExecutionDataPruner *pruner.Pruner ExecutionDataDatastore *badger.Datastore ExecutionDataTracker tracker.Storage + versionControl *version.VersionControl // The sync engine participants provider is the libp2p peer store for the access node // which is not available until after the network has started. @@ -1226,6 +1231,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { "circuit-breaker-max-requests", defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, "maximum number of requests to check if connection restored after timeout. Default value is 1") + flags.BoolVar(&builder.versionControlEnabled, + "version-control-enabled", + defaultConfig.versionControlEnabled, + "whether to enable the version control feature. Default value is true") // ExecutionDataRequester config flags.BoolVar(&builder.executionDataSyncEnabled, "execution-data-sync-enabled", @@ -1937,6 +1946,34 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return builder.RequestEng, nil }) + if builder.versionControlEnabled { + builder.Component("version control", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + nodeVersion, err := build.Semver() + if err != nil { + return nil, fmt.Errorf("could not load node version for version control. "+ + "version (%s) is not semver compliant: %w. Make sure a valid semantic version is provided in the VERSION environment variable", build.Version(), err) + } + + versionControl, err := version.NewVersionControl( + builder.Logger, + node.Storage.VersionBeacons, + nodeVersion, + builder.SealedRootBlock.Header.Height, + builder.LastFinalizedHeader.Height, + ) + if err != nil { + return nil, fmt.Errorf("could not create version control: %w", err) + } + + // VersionControl needs to consume BlockFinalized events. + node.ProtocolEvents.AddConsumer(versionControl) + + builder.versionControl = versionControl + + return versionControl, nil + }) + } + if builder.supportsObserver { builder.Component("public sync request handler", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { syncRequestHandler, err := synceng.NewRequestHandlerEngine( diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index d7eae4c133b..066559858f2 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -28,6 +28,7 @@ import ( "github.com/onflow/flow-go/admin/commands" stateSyncCommands "github.com/onflow/flow-go/admin/commands/state_synchronization" "github.com/onflow/flow-go/cmd" + "github.com/onflow/flow-go/cmd/build" "github.com/onflow/flow-go/consensus" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/committees" @@ -51,6 +52,7 @@ import ( "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/follower" synceng "github.com/onflow/flow-go/engine/common/synchronization" + "github.com/onflow/flow-go/engine/common/version" "github.com/onflow/flow-go/engine/execution/computation/query" "github.com/onflow/flow-go/fvm/storage/derived" "github.com/onflow/flow-go/ledger" @@ -155,6 +157,7 @@ type ObserverServiceConfig struct { executionDataPrunerHeightRangeTarget uint64 executionDataPrunerThreshold uint64 localServiceAPIEnabled bool + versionControlEnabled bool executionDataDir string executionDataStartHeight uint64 executionDataConfig edrequester.ExecutionDataConfig @@ -227,6 +230,7 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { executionDataPrunerHeightRangeTarget: 0, executionDataPrunerThreshold: 100_000, localServiceAPIEnabled: false, + versionControlEnabled: true, executionDataDir: filepath.Join(homedir, ".flow", "execution_data"), executionDataStartHeight: 0, executionDataConfig: edrequester.ExecutionDataConfig{ @@ -267,6 +271,7 @@ type ObserverServiceBuilder struct { ExecutionIndexerCore *indexer.IndexerCore TxResultsIndex *index.TransactionResultsIndex IndexerDependencies *cmd.DependencyList + versionControl *version.VersionControl ExecutionDataDownloader execution_data.Downloader ExecutionDataRequester state_synchronization.ExecutionDataRequester @@ -664,6 +669,10 @@ func (builder *ObserverServiceBuilder) extraFlags() { "execution-data-indexing-enabled", defaultConfig.executionDataIndexingEnabled, "whether to enable the execution data indexing") + flags.BoolVar(&builder.versionControlEnabled, + "version-control-enabled", + defaultConfig.versionControlEnabled, + "whether to enable the version control feature. Default value is true") flags.BoolVar(&builder.localServiceAPIEnabled, "local-service-api-enabled", defaultConfig.localServiceAPIEnabled, "whether to use local indexed data for api queries") flags.StringVar(&builder.registersDBPath, "execution-state-dir", defaultConfig.registersDBPath, "directory to use for execution-state database") flags.StringVar(&builder.checkpointFile, "execution-state-checkpoint", defaultConfig.checkpointFile, "execution-state checkpoint file") @@ -1685,7 +1694,6 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { } func (builder *ObserverServiceBuilder) enqueueRPCServer() { - builder.Module("transaction metrics", func(node *cmd.NodeConfig) error { var err error builder.TransactionTimings, err = stdmap.NewTransactionTimings(1500 * 300) // assume 1500 TPS * 300 seconds @@ -1783,6 +1791,33 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.ScriptExecutor = backend.NewScriptExecutor(builder.Logger, builder.scriptExecMinBlock, builder.scriptExecMaxBlock) return nil }) + if builder.versionControlEnabled { + builder.Component("version control", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + nodeVersion, err := build.Semver() + if err != nil { + return nil, fmt.Errorf("could not load node version for version control. "+ + "version (%s) is not semver compliant: %w. Make sure a valid semantic version is provided in the VERSION environment variable", build.Version(), err) + } + + versionControl, err := version.NewVersionControl( + builder.Logger, + node.Storage.VersionBeacons, + nodeVersion, + builder.SealedRootBlock.Header.Height, + builder.LastFinalizedHeader.Height, + ) + if err != nil { + return nil, fmt.Errorf("could not create version control: %w", err) + } + + // VersionControl needs to consume BlockFinalized events. + node.ProtocolEvents.AddConsumer(versionControl) + + builder.versionControl = versionControl + + return versionControl, nil + }) + } builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { accessMetrics := builder.AccessMetrics config := builder.rpcConf diff --git a/engine/common/version/version_control.go b/engine/common/version/version_control.go new file mode 100644 index 00000000000..cda72e537b2 --- /dev/null +++ b/engine/common/version/version_control.go @@ -0,0 +1,367 @@ +package version + +import ( + "errors" + "fmt" + "sync" + + "github.com/coreos/go-semver/semver" + "github.com/rs/zerolog" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/counters" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/state/protocol" + psEvents "github.com/onflow/flow-go/state/protocol/events" + "github.com/onflow/flow-go/storage" +) + +// ErrOutOfRange indicates that height is higher than last handled block height +var ErrOutOfRange = errors.New("height is out of range") + +// VersionControlConsumer defines a function type that consumes version control updates. +// It is called with the block height and the corresponding semantic version. +// There are two possible notifications options: +// - A new or updated version will have a height and a semantic version at that height. +// - A deleted version will have the previous height and nil semantic version, indicating that the update was deleted. +type VersionControlConsumer func(height uint64, version *semver.Version) + +// NoHeight represents the maximum possible height for blocks. +var NoHeight = uint64(0) + +// VersionControl manages the version control system for the node. +// It consumes BlockFinalized events and updates the node's version control based on the latest version beacon. +// +// VersionControl implements the protocol.Consumer and component.Component interfaces. +type VersionControl struct { + // Noop implements the protocol.Consumer interface with no operations. + psEvents.Noop + sync.Mutex + component.Component + + log zerolog.Logger + // Storage + versionBeacons storage.VersionBeacons + + // nodeVersion stores the node's current version. + // It could be nil if the node version is not available. + nodeVersion *semver.Version + + // consumers stores the list of consumers for version updates. + consumers []VersionControlConsumer + + // Notifier for new finalized block height + finalizedHeightNotifier engine.Notifier + + finalizedHeight counters.StrictMonotonousCounter + + // lastProcessedHeight the last handled block height + lastProcessedHeight *atomic.Uint64 + + // sealedRootBlockHeight the last sealed block height when node bootstrapped + sealedRootBlockHeight *atomic.Uint64 + + // startHeight and endHeight define the height boundaries for version compatibility. + startHeight *atomic.Uint64 + endHeight *atomic.Uint64 +} + +var _ protocol.Consumer = (*VersionControl)(nil) +var _ component.Component = (*VersionControl)(nil) + +// NewVersionControl creates a new VersionControl instance. +// +// We currently have no strong guarantee that the node version is a valid semver. +// See build.SemverV2 for more details. That is why nil is a valid input for node version. +func NewVersionControl( + log zerolog.Logger, + versionBeacons storage.VersionBeacons, + nodeVersion *semver.Version, + sealedRootBlockHeight uint64, + latestFinalizedBlockHeight uint64, +) (*VersionControl, error) { + + vc := &VersionControl{ + log: log.With(). + Str("component", "version_control"). + Logger(), + + nodeVersion: nodeVersion, + versionBeacons: versionBeacons, + sealedRootBlockHeight: atomic.NewUint64(sealedRootBlockHeight), + lastProcessedHeight: atomic.NewUint64(latestFinalizedBlockHeight), + finalizedHeight: counters.NewMonotonousCounter(latestFinalizedBlockHeight), + finalizedHeightNotifier: engine.NewNotifier(), + startHeight: atomic.NewUint64(NoHeight), + endHeight: atomic.NewUint64(NoHeight), + } + + if vc.nodeVersion == nil { + return nil, fmt.Errorf("version control node version is empty") + } + + vc.log.Info(). + Stringer("node_version", vc.nodeVersion). + Msg("system initialized") + + // Setup component manager for handling worker functions. + cm := component.NewComponentManagerBuilder() + cm.AddWorker(vc.processEvents) + cm.AddWorker(vc.checkInitialVersionBeacon) + + vc.Component = cm.Build() + + return vc, nil +} + +// checkInitialVersionBeacon checks the initial version beacon at the latest finalized block. +// It ensures the component is not ready until the initial version beacon is checked. +func (v *VersionControl) checkInitialVersionBeacon( + ctx irrecoverable.SignalerContext, + ready component.ReadyFunc, +) { + err := v.initBoundaries(ctx) + if err == nil { + ready() + } +} + +// initBoundaries initializes the version boundaries for version control. +// +// It searches through version beacons to find the start and end block heights +// for the current node version. The search continues until the start height +// is found or until the sealed root block height is reached. +// +// Returns an error when could not get the highest version beacon event +func (v *VersionControl) initBoundaries( + ctx irrecoverable.SignalerContext, +) error { + sealedRootBlockHeight := v.sealedRootBlockHeight.Load() + latestHeight := v.lastProcessedHeight.Load() + processedHeight := latestHeight + + for { + vb, err := v.versionBeacons.Highest(processedHeight) + if err != nil && !errors.Is(err, storage.ErrNotFound) { + ctx.Throw( + fmt.Errorf( + "failed to get highest version beacon for version control: %w", + err)) + return err + } + + if vb == nil { + // no version beacon found + // this is unexpected on a live network as there should always be at least the + // starting version beacon, but not fatal. + // It can happen on new or test networks if the node starts before bootstrap is finished. + // TODO: remove when we can guarantee that there will always be a version beacon + v.log.Info(). + Uint64("height", processedHeight). + Msg("No version beacon found for version control") + + return nil + } + + // version boundaries are sorted by blockHeight in ascending order + // the first version greater than the node's is the version transition height + for i := len(vb.VersionBoundaries) - 1; i >= 0; i-- { + boundary := vb.VersionBoundaries[i] + + ver, err := boundary.Semver() + // this should never happen as we already validated the version beacon + // when indexing it + if err != nil || ver == nil { + if err == nil { + err = fmt.Errorf("boundary semantic version is nil") + } + ctx.Throw( + fmt.Errorf( + "failed to parse semver during version control setup: %w", + err)) + return err + } + + compResult := ver.Compare(*v.nodeVersion) + processedHeight = vb.SealHeight - 1 + + if compResult <= 0 { + v.startHeight.Store(boundary.BlockHeight) + v.log.Info(). + Uint64("startHeight", boundary.BlockHeight). + Msg("Found start block height") + // This is the lowest compatible height for this node version, stop search immediately + return nil + } else { + v.endHeight.Store(boundary.BlockHeight - 1) + v.log.Info(). + Uint64("endHeight", boundary.BlockHeight-1). + Msg("Found end block height") + } + } + + // The search should continue until we find the start height or reach the sealed root block height + if v.startHeight.Load() == NoHeight && processedHeight <= sealedRootBlockHeight { + v.log.Info(). + Uint64("processedHeight", processedHeight). + Uint64("sealedRootBlockHeight", sealedRootBlockHeight). + Msg("No start version beacon event found") + return nil + } + } +} + +// BlockFinalized is called when a block is finalized. +// It implements the protocol.Consumer interface. +func (v *VersionControl) BlockFinalized(h *flow.Header) { + if v.finalizedHeight.Set(h.Height) { + v.finalizedHeightNotifier.Notify() + } +} + +// CompatibleAtBlock checks if the node's version is compatible at a given block height. +// It returns true if the node's version is compatible within the specified height range. +// Returns expected errors: +// - ErrOutOfRange if incoming block height is higher that last handled block height +func (v *VersionControl) CompatibleAtBlock(height uint64) (bool, error) { + // Check, if the height smaller than sealed root block height. If so, return an error indicating that the height is unhandled. + sealedRootHeight := v.sealedRootBlockHeight.Load() + if height < sealedRootHeight { + return false, fmt.Errorf("could not check compatibility for height %d: the provided height is smaller than sealed root height %d: %w", height, sealedRootHeight, ErrOutOfRange) + } + + // Check if the height is greater than the last handled block height. If so, return an error indicating that the height is unhandled. + lastProcessedHeight := v.lastProcessedHeight.Load() + if height > lastProcessedHeight { + return false, fmt.Errorf("could not check compatibility for height %d: last handled height is %d: %w", height, lastProcessedHeight, ErrOutOfRange) + } + + startHeight := v.startHeight.Load() + // Check if the start height is set and the height is less than the start height. If so, return false indicating that the height is not compatible. + if startHeight != NoHeight && height < startHeight { + return false, nil + } + + endHeight := v.endHeight.Load() + // Check if the end height is set and the height is greater than the end height. If so, return false indicating that the height is not compatible. + if endHeight != NoHeight && height > endHeight { + return false, nil + } + + // If none of the above conditions are met, the height is compatible. + return true, nil +} + +// AddVersionUpdatesConsumer adds a consumer for version update events. +func (v *VersionControl) AddVersionUpdatesConsumer(consumer VersionControlConsumer) { + v.Lock() + defer v.Unlock() + + v.consumers = append(v.consumers, consumer) +} + +// processEvents is a worker that processes block finalized events. +func (v *VersionControl) processEvents( + ctx irrecoverable.SignalerContext, + ready component.ReadyFunc, +) { + ready() + + for { + select { + case <-ctx.Done(): + return + case <-v.finalizedHeightNotifier.Channel(): + v.blockFinalized(ctx, v.finalizedHeight.Value()) + } + } +} + +// blockFinalized processes a block finalized event and updates the version control state. +func (v *VersionControl) blockFinalized( + ctx irrecoverable.SignalerContext, + newFinalizedHeight uint64, +) { + lastProcessedHeight := v.lastProcessedHeight.Load() + if lastProcessedHeight >= newFinalizedHeight { + // already processed this or a higher version beacon + return + } + + for height := lastProcessedHeight + 1; height <= newFinalizedHeight; height++ { + vb, err := v.versionBeacons.Highest(height) + if err != nil { + v.log.Err(err). + Uint64("height", height). + Msg("Failed to get highest version beacon for version control") + + ctx.Throw( + fmt.Errorf( + "failed to get highest version beacon for version control: %w", + err)) + return + } + + if vb == nil { + // no version beacon found + // this is unexpected as there should always be at least the + // starting version beacon, but not fatal. + // It can happen if the node starts before bootstrap is finished. + // TODO: remove when we can guarantee that there will always be a version beacon + v.log.Info(). + Uint64("height", height). + Msg("No version beacon found for version control") + continue + } + + v.lastProcessedHeight.Store(height) + + previousEndHeight := v.endHeight.Load() + + if previousEndHeight != NoHeight && height > previousEndHeight { + // Stop here since it's outside our compatible range + return + } + + newEndHeight := NoHeight + // version boundaries are sorted by blockHeight in ascending order + for _, boundary := range vb.VersionBoundaries { + ver, err := boundary.Semver() + if err != nil || ver == nil { + if err == nil { + err = fmt.Errorf("boundary semantic version is nil") + } + // this should never happen as we already validated the version beacon + // when indexing it + ctx.Throw( + fmt.Errorf( + "failed to parse semver: %w", + err)) + return + } + + if ver.Compare(*v.nodeVersion) > 0 { + newEndHeight = boundary.BlockHeight - 1 + + for _, consumer := range v.consumers { + consumer(boundary.BlockHeight, ver) + } + + break + } + } + + v.endHeight.Store(newEndHeight) + + // Check if previous version was deleted. If yes, notify consumers about deletion + if previousEndHeight != NoHeight && newEndHeight == NoHeight { + for _, consumer := range v.consumers { + // Note: notifying for the boundary height, which is end height + 1 + consumer(previousEndHeight+1, nil) + } + } + } +} diff --git a/engine/common/version/version_control_test.go b/engine/common/version/version_control_test.go new file mode 100644 index 00000000000..c6917076a79 --- /dev/null +++ b/engine/common/version/version_control_test.go @@ -0,0 +1,608 @@ +package version + +import ( + "context" + "errors" + "fmt" + "math" + "sort" + "testing" + "time" + + "github.com/onflow/flow-go/utils/unittest/mocks" + + "github.com/coreos/go-semver/semver" + "github.com/stretchr/testify/assert" + testifyMock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/storage" + storageMock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +// testCaseConfig contains custom tweaks for test cases +type testCaseConfig struct { + name string + nodeVersion string + + versionEvents []*flow.SealedVersionBeacon + expectedStart uint64 + expectedEnd uint64 +} + +// TestVersionControlInitialization tests the initialization process of the VersionControl component +func TestVersionControlInitialization(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sealedRootBlockHeight := uint64(1000) + latestBlockHeight := sealedRootBlockHeight + 100 + + testCases := []testCaseConfig{ + { + name: "no version beacon found", + nodeVersion: "0.0.1", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight-100, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight - 50, Version: "0.0.1"}), + }, + expectedStart: sealedRootBlockHeight, + expectedEnd: latestBlockHeight, + }, + { + name: "start version set", + nodeVersion: "0.0.1", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.1"}), + }, + expectedStart: sealedRootBlockHeight + 12, + expectedEnd: latestBlockHeight, + }, + { + name: "correct start version found", + nodeVersion: "0.0.3", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight+2, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 4, Version: "0.0.1"}), + VersionBeaconEvent(sealedRootBlockHeight+5, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 7, Version: "0.0.2"}), + }, + expectedStart: sealedRootBlockHeight + 7, + expectedEnd: latestBlockHeight, + }, + { + name: "end version set", + nodeVersion: "0.0.1", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight-100, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight - 50, Version: "0.0.1"}), + VersionBeaconEvent(latestBlockHeight-10, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 8, Version: "0.0.3"}), + }, + expectedStart: sealedRootBlockHeight, + expectedEnd: latestBlockHeight - 9, + }, + { + name: "correct end version found", + nodeVersion: "0.0.1", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight-100, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight - 50, Version: "0.0.1"}), + VersionBeaconEvent(latestBlockHeight-10, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 8, Version: "0.0.3"}), + VersionBeaconEvent(latestBlockHeight-3, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 1, Version: "0.0.4"}), + }, + expectedStart: sealedRootBlockHeight, + expectedEnd: latestBlockHeight - 9, + }, + { + name: "start and end version set", + nodeVersion: "0.0.2", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.1"}), + VersionBeaconEvent(latestBlockHeight-10, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 8, Version: "0.0.3"}), + }, + expectedStart: sealedRootBlockHeight + 12, + expectedEnd: latestBlockHeight - 9, + }, + { + name: "correct start and end version found", + nodeVersion: "0.0.2", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight+2, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 4, Version: "0.0.1"}), + VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.2"}), + VersionBeaconEvent(latestBlockHeight-10, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 8, Version: "0.0.3"}), + VersionBeaconEvent(latestBlockHeight-3, + flow.VersionBoundary{BlockHeight: latestBlockHeight - 1, Version: "0.0.4"}), + }, + expectedStart: sealedRootBlockHeight + 12, + expectedEnd: latestBlockHeight - 9, + }, + { + name: "node's version is too old for current latest", + nodeVersion: "0.0.1", + versionEvents: []*flow.SealedVersionBeacon{ + // the node's version is too old for the earliest version boundary for the network + VersionBeaconEvent(sealedRootBlockHeight-100, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight - 50, Version: "0.0.2"}), + }, + expectedStart: math.MaxUint64, + expectedEnd: math.MaxUint64, + }, + { + name: "node's version is too new for current latest", + nodeVersion: "0.0.3", + versionEvents: []*flow.SealedVersionBeacon{ + VersionBeaconEvent(sealedRootBlockHeight-100, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight - 50, Version: "0.0.2"}), + + // the version boundary that transitions to the node's version applies after the + // latest finalized block, so the node's version is not compatible with any block + VersionBeaconEvent(latestBlockHeight-3, + flow.VersionBoundary{BlockHeight: latestBlockHeight + 1, Version: "0.0.3"}), + VersionBeaconEvent(latestBlockHeight-2, + flow.VersionBoundary{BlockHeight: latestBlockHeight + 2, Version: "0.0.4"}), + }, + expectedStart: math.MaxUint64, + expectedEnd: math.MaxUint64, + }, + { + name: "pre-release versions handled as expected", + nodeVersion: "0.0.1-pre-release.1", + versionEvents: []*flow.SealedVersionBeacon{ + // 0.0.1-pre-release.1 > 0.0.1-pre-release.0 + VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.1-pre-release.0"}), + // 0.0.1-pre-release.1 < 0.0.1 + VersionBeaconEvent(sealedRootBlockHeight+12, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 14, Version: "0.0.1"}), + }, + expectedStart: sealedRootBlockHeight + 12, + expectedEnd: sealedRootBlockHeight + 13, + }, + { + name: "0.0.0 handled as expected", + nodeVersion: "0.0.0-20230101000000-c0c9f774e40c", + versionEvents: []*flow.SealedVersionBeacon{ + // 0.0.0-20230101000000-c0c9f774e40c > 0.0.0-20220101000000-7b4eea64cf58 + VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.0-20220101000000-7b4eea64cf58"}), + // 0.0.0-20230101000000-c0c9f774e40c < 0.0.0-20240101000000-6ceb2ff114de + VersionBeaconEvent(sealedRootBlockHeight+12, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 14, Version: "0.0.0-20240101000000-6ceb2ff114de"}), + }, + expectedStart: sealedRootBlockHeight + 12, + expectedEnd: sealedRootBlockHeight + 13, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + eventMap := make(map[uint64]*flow.SealedVersionBeacon, len(testCase.versionEvents)) + for _, event := range testCase.versionEvents { + eventMap[event.SealHeight] = event + } + + // make sure events are sorted descending by seal height + sort.Slice(testCase.versionEvents, func(i, j int) bool { + return testCase.versionEvents[i].SealHeight > testCase.versionEvents[j].SealHeight + }) + + versionBeacons := storageMock.NewVersionBeacons(t) + versionBeacons. + On("Highest", testifyMock.AnythingOfType("uint64")). + Return(func(height uint64) (*flow.SealedVersionBeacon, error) { + // iterating through events sorted descending by seal height + // return the first event that was sealed in a height less than or equal to height + for _, event := range testCase.versionEvents { + if event.SealHeight <= height { + return event, nil + } + } + return nil, storage.ErrNotFound + }) + + vc := createVersionControlComponent(t, versionComponentTestConfigs{ + nodeVersion: testCase.nodeVersion, + versionBeacons: versionBeacons, + sealedRootBlockHeight: sealedRootBlockHeight, + latestFinalizedBlockHeight: latestBlockHeight, + signalerContext: irrecoverable.NewMockSignalerContext(t, ctx), + }) + + checks := generateChecks(testCase, sealedRootBlockHeight, latestBlockHeight) + + for height, expected := range checks { + compatible, err := vc.CompatibleAtBlock(height) + + require.NoError(t, err) + assert.Equal(t, expected, compatible, "unexpected compatibility at height %d. want: %t, got %t", height, expected, compatible) + } + }) + } +} + +// TestVersionControlInitializationWithErrors tests the initialization process of the VersionControl component with error cases +func TestVersionControlInitializationWithErrors(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sealedRootBlockHeight := uint64(1000) + latestBlockHeight := sealedRootBlockHeight + 100 + eventMap := map[uint64]*flow.SealedVersionBeacon{ + sealedRootBlockHeight + 10: VersionBeaconEvent(sealedRootBlockHeight+10, + flow.VersionBoundary{BlockHeight: sealedRootBlockHeight + 12, Version: "0.0.1"}), + } + + versionBeacons := storageMock.NewVersionBeacons(t) + + checkForError := func(height uint64) { + versionBeacons. + On("Highest", testifyMock.AnythingOfType("uint64")). + Return(mocks.StorageMapGetter(eventMap)).Once() + + vc := createVersionControlComponent(t, versionComponentTestConfigs{ + nodeVersion: "0.0.1", + versionBeacons: versionBeacons, + sealedRootBlockHeight: sealedRootBlockHeight, + latestFinalizedBlockHeight: latestBlockHeight, + signalerContext: irrecoverable.NewMockSignalerContext(t, ctx), + }) + + compatible, err := vc.CompatibleAtBlock(height) + + assert.True(t, errors.Is(err, ErrOutOfRange)) + assert.False(t, compatible) + } + + t.Run("height is bigger than latest block height", func(t *testing.T) { + checkForError(latestBlockHeight + 1) + }) + + t.Run("height is smaller than sealed root block height", func(t *testing.T) { + checkForError(sealedRootBlockHeight - 1) + }) + + t.Run("failed to complete initialization due to \"Highest\" of version beacon returns error", func(t *testing.T) { + decodeErr := fmt.Errorf("test decode error") + + versionBeacons. + On("Highest", testifyMock.AnythingOfType("uint64")). + Return(nil, decodeErr).Once() + + vc, err := NewVersionControl( + unittest.Logger(), + versionBeacons, + semver.New("0.0.1"), + sealedRootBlockHeight, + latestBlockHeight, + ) + require.NoError(t, err) + + vc.Start(irrecoverable.NewMockSignalerContextExpectError(t, ctx, fmt.Errorf( + "failed to get highest version beacon for version control: %w", + decodeErr))) + + unittest.AssertNotClosesBefore(t, vc.Ready(), 2*time.Second) + }) +} + +func generateChecks(testCase testCaseConfig, finalizedRootBlockHeight, latestBlockHeight uint64) map[uint64]bool { + checks := map[uint64]bool{} + if testCase.expectedStart == math.MaxUint64 && testCase.expectedEnd == math.MaxUint64 { + for height := finalizedRootBlockHeight; height <= latestBlockHeight; height++ { + checks[height] = false + } + return checks + } + + checks[testCase.expectedStart] = true + checks[testCase.expectedEnd] = true + + if testCase.expectedStart > finalizedRootBlockHeight { + checks[finalizedRootBlockHeight] = false + checks[testCase.expectedStart-1] = false + } + + if testCase.expectedEnd < latestBlockHeight { + checks[latestBlockHeight] = false + checks[testCase.expectedEnd+1] = false + } + + return checks +} + +// TestVersionBoundaryUpdated tests the behavior of the VersionControl component when the version is updated. +func TestVersionBoundaryUpdated(t *testing.T) { + signalCtx := irrecoverable.NewMockSignalerContext(t, context.Background()) + + contract := &versionBeaconContract{} + + // Create version event for initial height + latestHeight := uint64(10) + boundaryHeight := uint64(13) + + vc := createVersionControlComponent(t, versionComponentTestConfigs{ + nodeVersion: "0.0.1", + versionBeacons: contract, + sealedRootBlockHeight: 0, + latestFinalizedBlockHeight: latestHeight, + signalerContext: signalCtx, + }) + + var assertUpdate func(height uint64, version *semver.Version) + var assertCallbackCalled func() + + // Add a consumer to verify version updates + vc.AddVersionUpdatesConsumer(func(height uint64, version *semver.Version) { + assertUpdate(height, version) + }) + assert.Len(t, vc.consumers, 1) + + // At this point, both start and end heights are unset + + // Add a new boundary, and finalize the block + latestHeight++ // 11 + contract.AddBoundary(latestHeight, flow.VersionBoundary{BlockHeight: boundaryHeight, Version: "0.0.2"}) + + assertUpdate, assertCallbackCalled = generateConsumerAssertions(t, boundaryHeight, semver.New("0.0.2")) + vc.blockFinalized(signalCtx, latestHeight) + assertCallbackCalled() + + // Next, update the boundary and finalize the block + latestHeight++ // 12 + contract.UpdateBoundary(latestHeight, boundaryHeight, "0.0.3") + + assertUpdate, assertCallbackCalled = generateConsumerAssertions(t, boundaryHeight, semver.New("0.0.3")) + vc.blockFinalized(signalCtx, latestHeight) + assertCallbackCalled() + + // Finally, finalize one more block to get past the boundary + latestHeight++ // 13 + vc.blockFinalized(signalCtx, latestHeight) + + // Check compatibility at various heights + compatible, err := vc.CompatibleAtBlock(10) + require.NoError(t, err) + assert.True(t, compatible) + + compatible, err = vc.CompatibleAtBlock(12) + require.NoError(t, err) + assert.True(t, compatible) + + compatible, err = vc.CompatibleAtBlock(13) + require.NoError(t, err) + assert.False(t, compatible) +} + +// TestVersionBoundaryDeleted tests the behavior of the VersionControl component when the version is deleted. +func TestVersionBoundaryDeleted(t *testing.T) { + signalCtx := irrecoverable.NewMockSignalerContext(t, context.Background()) + + contract := &versionBeaconContract{} + + // Create version event for initial height + latestHeight := uint64(10) + boundaryHeight := uint64(13) + + vc := createVersionControlComponent(t, versionComponentTestConfigs{ + nodeVersion: "0.0.1", + versionBeacons: contract, + sealedRootBlockHeight: 0, + latestFinalizedBlockHeight: latestHeight, + signalerContext: signalCtx, + }) + + var assertUpdate func(height uint64, version *semver.Version) + var assertCallbackCalled func() + + // Add a consumer to verify version updates + vc.AddVersionUpdatesConsumer(func(height uint64, version *semver.Version) { + assertUpdate(height, version) + }) + assert.Len(t, vc.consumers, 1) + + // Add a new boundary, and finalize the block + latestHeight++ // 11 + contract.AddBoundary(latestHeight, flow.VersionBoundary{BlockHeight: boundaryHeight, Version: "0.0.2"}) + + assertUpdate, assertCallbackCalled = generateConsumerAssertions(t, boundaryHeight, semver.New("0.0.2")) + vc.blockFinalized(signalCtx, latestHeight) + assertCallbackCalled() + + // Next, delete the boundary and finalize the block + latestHeight++ // 12 + contract.DeleteBoundary(latestHeight, boundaryHeight) + + assertUpdate, assertCallbackCalled = generateConsumerAssertions(t, boundaryHeight, nil) // called with empty string signalling deleted + vc.blockFinalized(signalCtx, latestHeight) + assertCallbackCalled() + + // Finally, finalize one more block to get past the boundary + latestHeight++ // 13 + vc.blockFinalized(signalCtx, latestHeight) + + // Check compatibility at various heights + compatible, err := vc.CompatibleAtBlock(10) + require.NoError(t, err) + assert.True(t, compatible) + + compatible, err = vc.CompatibleAtBlock(12) + require.NoError(t, err) + assert.True(t, compatible) + + compatible, err = vc.CompatibleAtBlock(13) + require.NoError(t, err) + assert.True(t, compatible) +} + +// TestNotificationSkippedForCompatibleVersions tests that the VersionControl component does not +// send notifications to consumers VersionBeacon events with compatible versions. +func TestNotificationSkippedForCompatibleVersions(t *testing.T) { + signalCtx := irrecoverable.NewMockSignalerContext(t, context.Background()) + + contract := &versionBeaconContract{} + + // Create version event for initial height + latestHeight := uint64(10) + boundaryHeight := uint64(13) + + vc := createVersionControlComponent(t, versionComponentTestConfigs{ + nodeVersion: "0.0.1", + versionBeacons: contract, + sealedRootBlockHeight: 0, + latestFinalizedBlockHeight: latestHeight, + signalerContext: signalCtx, + }) + + // Add a consumer to verify notification is never sent + vc.AddVersionUpdatesConsumer(func(height uint64, version *semver.Version) { + t.Errorf("unexpected callback called at height %d with version %s", height, version) + }) + assert.Len(t, vc.consumers, 1) + + // Add a new boundary, and finalize the block + latestHeight++ // 11 + contract.AddBoundary(latestHeight, flow.VersionBoundary{BlockHeight: boundaryHeight, Version: "0.0.1-pre-release"}) + + vc.blockFinalized(signalCtx, latestHeight) + + // Check compatibility at various heights + compatible, err := vc.CompatibleAtBlock(10) + require.NoError(t, err) + assert.True(t, compatible) + + compatible, err = vc.CompatibleAtBlock(11) + require.NoError(t, err) + assert.True(t, compatible) +} + +func generateConsumerAssertions( + t *testing.T, + boundaryHeight uint64, + version *semver.Version, +) (func(height uint64, semver *semver.Version), func()) { + called := false + + assertUpdate := func(height uint64, semver *semver.Version) { + assert.Equal(t, boundaryHeight, height) + assert.Equal(t, version, semver) + called = true + } + + assertCalled := func() { + assert.True(t, called) + } + + return assertUpdate, assertCalled +} + +// versionComponentTestConfigs contains custom tweaks for version control creation +type versionComponentTestConfigs struct { + nodeVersion string + versionBeacons storage.VersionBeacons + sealedRootBlockHeight uint64 + latestFinalizedBlockHeight uint64 + signalerContext *irrecoverable.MockSignalerContext +} + +func createVersionControlComponent( + t *testing.T, + config versionComponentTestConfigs, +) *VersionControl { + // Create a new VersionControl instance with initial parameters. + vc, err := NewVersionControl( + unittest.Logger(), + config.versionBeacons, + semver.New(config.nodeVersion), + config.sealedRootBlockHeight, + config.latestFinalizedBlockHeight, + ) + require.NoError(t, err) + + // Start the VersionControl component. + vc.Start(config.signalerContext) + unittest.RequireComponentsReadyBefore(t, 2*time.Second, vc) + + return vc +} + +// VersionBeaconEvent creates a SealedVersionBeacon for the given heights and versions. +func VersionBeaconEvent(sealHeight uint64, vb ...flow.VersionBoundary) *flow.SealedVersionBeacon { + return &flow.SealedVersionBeacon{ + VersionBeacon: unittest.VersionBeaconFixture( + unittest.WithBoundaries(vb...), + ), + SealHeight: sealHeight, + } +} + +type versionBeaconContract struct { + boundaries []flow.VersionBoundary + events []*flow.SealedVersionBeacon +} + +func (c *versionBeaconContract) Highest(belowOrEqualTo uint64) (*flow.SealedVersionBeacon, error) { + for _, event := range c.events { + if event.SealHeight <= belowOrEqualTo { + return event, nil + } + } + return nil, storage.ErrNotFound +} + +func (c *versionBeaconContract) AddBoundary(sealedHeight uint64, boundary flow.VersionBoundary) { + c.boundaries = append(c.boundaries, boundary) + c.emitEvent(sealedHeight) +} + +func (c *versionBeaconContract) DeleteBoundary(sealedHeight, boundaryHeight uint64) { + for i, boundary := range c.boundaries { + if boundary.BlockHeight == boundaryHeight { + c.boundaries = append(c.boundaries[:i], c.boundaries[i+1:]...) + break + } + } + c.emitEvent(sealedHeight) +} + +func (c *versionBeaconContract) UpdateBoundary(sealedHeight, boundaryHeight uint64, version string) { + for i, boundary := range c.boundaries { + if boundary.BlockHeight == boundaryHeight { + c.boundaries[i].Version = version + break + } + } + c.emitEvent(sealedHeight) +} + +func (c *versionBeaconContract) emitEvent(sealedHeight uint64) { + // sort boundaries ascending by height + sort.Slice(c.boundaries, func(i, j int) bool { + return c.boundaries[i].BlockHeight < c.boundaries[j].BlockHeight + }) + + // include only future boundaries + boundaries := make([]flow.VersionBoundary, 0) + for _, boundary := range c.boundaries { + if boundary.BlockHeight >= sealedHeight { + boundaries = append(boundaries, boundary) + } + } + c.events = append(c.events, VersionBeaconEvent(sealedHeight, boundaries...)) + + // sort boundaries descending by height + sort.Slice(c.events, func(i, j int) bool { + return c.events[i].SealHeight > c.events[j].SealHeight + }) +}