-
Notifications
You must be signed in to change notification settings - Fork 972
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR introduces `full` and `bridge` node pruning via the `--experimental-pruning` flag. Support is included for nodes that start from scratch with pruning enabled and also for `archival` (nodes retaining all historical blocks) that enable the `--experimental-pruning` flag. _Note that this PR does not support the conversion of a pruned node into an archival one explicitly (it would not support re-syncing deleted blocks)._ With pruning enabled, `full` and `bridge` nodes' block stores can be expected not to exceed ~4TB (as the upper bound). In follow-up PRs (hardening), the following features can be expected: - [x] discovery for archival nodes for archival sync - [ ] inverted_index / light node pruning - [ ] include more metrics for errors TODO: - [x] clean up some TODOs - [x] fix one flakey unit test - [x] change values back to the actual (GC cycle, sampling window, pruning window, etc). - [x] figure out whether to store error in pruner checkpoint - [x] fix issue with pruning genesis block via findPruneableHeaders - [x] metrics for failed prunes - [x] set a sane default for max pruneable / consider removing `MaxPruneablePerGC` as now context timeouts are on a per block basis - [ ] dedup findPruneableHeader test utility - [x] badger dep --------- Co-authored-by: Ryan <[email protected]>
- Loading branch information
1 parent
137a1ad
commit 02030bf
Showing
23 changed files
with
1,078 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package pruner | ||
|
|
||
| var MetricsEnabled bool | ||
|
|
||
| type Config struct { | ||
| EnableService bool | ||
| } | ||
|
|
||
| func DefaultConfig() Config { | ||
| return Config{ | ||
| EnableService: false, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| package pruner | ||
|
|
||
| import ( | ||
| "github.com/ipfs/go-datastore" | ||
|
|
||
| hdr "github.com/celestiaorg/go-header" | ||
|
|
||
| "github.com/celestiaorg/celestia-node/header" | ||
| "github.com/celestiaorg/celestia-node/nodebuilder/p2p" | ||
| "github.com/celestiaorg/celestia-node/pruner" | ||
| ) | ||
|
|
||
| func newPrunerService( | ||
| p pruner.Pruner, | ||
| window pruner.AvailabilityWindow, | ||
| getter hdr.Store[*header.ExtendedHeader], | ||
| ds datastore.Batching, | ||
| opts ...pruner.Option, | ||
| ) (*pruner.Service, error) { | ||
| serv, err := pruner.NewService(p, window, getter, ds, p2p.BlockTime, opts...) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if MetricsEnabled { | ||
| err := pruner.WithPrunerMetrics(serv) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| return serv, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package pruner | ||
|
|
||
| import ( | ||
| "github.com/spf13/cobra" | ||
| flag "github.com/spf13/pflag" | ||
| ) | ||
|
|
||
| const pruningFlag = "experimental-pruning" | ||
|
|
||
| func Flags() *flag.FlagSet { | ||
| flags := &flag.FlagSet{} | ||
|
|
||
| flags.Bool(pruningFlag, false, "EXPERIMENTAL: Enables pruning of blocks outside the pruning window.") | ||
|
|
||
| return flags | ||
| } | ||
|
|
||
| func ParseFlags(cmd *cobra.Command, cfg *Config) { | ||
| cfg.EnableService = cmd.Flag(pruningFlag).Changed | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package pruner | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "go.uber.org/fx" | ||
|
|
||
| "github.com/celestiaorg/celestia-node/nodebuilder/node" | ||
| "github.com/celestiaorg/celestia-node/pruner" | ||
| "github.com/celestiaorg/celestia-node/pruner/archival" | ||
| "github.com/celestiaorg/celestia-node/pruner/full" | ||
| "github.com/celestiaorg/celestia-node/pruner/light" | ||
| "github.com/celestiaorg/celestia-node/share/eds" | ||
| ) | ||
|
|
||
| func ConstructModule(tp node.Type, cfg *Config) fx.Option { | ||
| if !cfg.EnableService { | ||
| switch tp { | ||
| case node.Light: | ||
| // light nodes are still subject to sampling within window | ||
| // even if pruning is not enabled. | ||
| return fx.Supply(light.Window) | ||
| case node.Full, node.Bridge: | ||
| return fx.Supply(archival.Window) | ||
| default: | ||
| panic("unknown node type") | ||
| } | ||
| } | ||
|
|
||
| baseComponents := fx.Options( | ||
| fx.Provide(fx.Annotate( | ||
| newPrunerService, | ||
| fx.OnStart(func(ctx context.Context, p *pruner.Service) error { | ||
| return p.Start(ctx) | ||
| }), | ||
| fx.OnStop(func(ctx context.Context, p *pruner.Service) error { | ||
| return p.Stop(ctx) | ||
| }), | ||
| )), | ||
| // This is necessary to invoke the pruner service as independent thanks to a | ||
| // quirk in FX. | ||
| fx.Invoke(func(_ *pruner.Service) {}), | ||
| ) | ||
|
|
||
| switch tp { | ||
| case node.Full: | ||
| return fx.Module("prune", | ||
| baseComponents, | ||
| fx.Provide(func(store *eds.Store) pruner.Pruner { | ||
| return full.NewPruner(store) | ||
| }), | ||
| fx.Supply(full.Window), | ||
| ) | ||
| case node.Bridge: | ||
| return fx.Module("prune", | ||
| baseComponents, | ||
| fx.Provide(func(store *eds.Store) pruner.Pruner { | ||
| return full.NewPruner(store) | ||
| }), | ||
| fx.Supply(full.Window), | ||
| ) | ||
| // TODO: Eventually, light nodes will be capable of pruning samples | ||
| // in which case, this can be enabled. | ||
| case node.Light: | ||
| return fx.Module("prune", | ||
| fx.Supply(light.Window), | ||
| ) | ||
| default: | ||
| panic("unknown node type") | ||
| } | ||
| } |
Oops, something went wrong.