From b35555978d670f81f2996f63a7a92362d65cb6df Mon Sep 17 00:00:00 2001 From: Sergey Gorbunov Date: Fri, 31 Jan 2025 18:33:11 +0300 Subject: [PATCH] feat: Better self-service commands for DHT providing (#10677) * Add lastRun, NextRun, move reprovide cmd to routing. * acceleratedDHT logic * changelog * depend on latest boxo --------- Co-authored-by: guillaumemichel Co-authored-by: Guillaume Michel --- core/commands/bitswap.go | 33 ++----------------- core/commands/commands_test.go | 2 +- core/commands/routing.go | 28 ++++++++++++++++ core/commands/stat_provide.go | 24 +++++++++++--- docs/changelogs/v0.34.md | 10 ++++++ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +-- go.mod | 2 +- go.sum | 4 +-- .../delegated_routing_v1_http_proxy_test.go | 2 +- .../delegated_routing_v1_http_server_test.go | 2 +- test/cli/provider_test.go | 12 +++---- test/cli/routing_dht_test.go | 2 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +-- 15 files changed, 80 insertions(+), 53 deletions(-) diff --git a/core/commands/bitswap.go b/core/commands/bitswap.go index 7f7ab963651..f577c03e31c 100644 --- a/core/commands/bitswap.go +++ b/core/commands/bitswap.go @@ -21,10 +21,9 @@ var BitswapCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "stat": bitswapStatCmd, - "wantlist": showWantlistCmd, - "ledger": ledgerCmd, - "reprovide": reprovideCmd, + "stat": bitswapStatCmd, + "wantlist": showWantlistCmd, + "ledger": ledgerCmd, }, } @@ -200,29 +199,3 @@ prints the ledger associated with a given peer. }), }, } - -var reprovideCmd = &cmds.Command{ - Helptext: cmds.HelpText{ - Tagline: "Trigger reprovider.", - ShortDescription: ` -Trigger reprovider to announce our data to network. -`, - }, - Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - nd, err := cmdenv.GetNode(env) - if err != nil { - return err - } - - if !nd.IsOnline { - return ErrNotOnline - } - - err = nd.Provider.Reprovide(req.Context) - if err != nil { - return err - } - - return nil - }, -} diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index b04a5459b50..5faf54e5d73 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -20,7 +20,6 @@ func TestCommands(t *testing.T) { "/add", "/bitswap", "/bitswap/ledger", - "/bitswap/reprovide", "/bitswap/stat", "/bitswap/wantlist", "/block", @@ -72,6 +71,7 @@ func TestCommands(t *testing.T) { "/routing/findpeer", "/routing/findprovs", "/routing/provide", + "/routing/reprovide", "/diag", "/diag/cmds", "/diag/cmds/clear", diff --git a/core/commands/routing.go b/core/commands/routing.go index c284166c8d1..3528a94f5ea 100644 --- a/core/commands/routing.go +++ b/core/commands/routing.go @@ -42,6 +42,7 @@ var RoutingCmd = &cmds.Command{ "get": getValueRoutingCmd, "put": putValueRoutingCmd, "provide": provideRefRoutingCmd, + "reprovide": reprovideRoutingCmd, }, } @@ -235,6 +236,33 @@ var provideRefRoutingCmd = &cmds.Command{ Type: routing.QueryEvent{}, } +var reprovideRoutingCmd = &cmds.Command{ + Status: cmds.Experimental, + Helptext: cmds.HelpText{ + Tagline: "Trigger reprovider.", + ShortDescription: ` +Trigger reprovider to announce our data to network. +`, + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + if !nd.IsOnline { + return ErrNotOnline + } + + err = nd.Provider.Reprovide(req.Context) + if err != nil { + return err + } + + return nil + }, +} + func provideKeys(ctx context.Context, r routing.Routing, cids []cid.Cid) error { for _, c := range cids { err := r.Provide(ctx, c, true) diff --git a/core/commands/stat_provide.go b/core/commands/stat_provide.go index 6ee51e516fa..8a3dcff305a 100644 --- a/core/commands/stat_provide.go +++ b/core/commands/stat_provide.go @@ -10,9 +10,15 @@ import ( "github.com/ipfs/boxo/provider" cmds "github.com/ipfs/go-ipfs-cmds" "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/libp2p/go-libp2p-kad-dht/fullrt" "golang.org/x/exp/constraints" ) +type reprovideStats struct { + provider.ReproviderStats + fullRT bool +} + var statProvideCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Returns statistics about the node's (re)provider system.", @@ -38,32 +44,42 @@ This interface is not stable and may change from release to release. if err != nil { return err } + _, fullRT := nd.DHTClient.(*fullrt.FullRT) - if err := res.Emit(stats); err != nil { + if err := res.Emit(reprovideStats{stats, fullRT}); err != nil { return err } return nil }, Encoders: cmds.EncoderMap{ - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s *provider.ReproviderStats) error { + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, s reprovideStats) error { wtr := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) defer wtr.Flush() fmt.Fprintf(wtr, "TotalProvides:\t%s\n", humanNumber(s.TotalProvides)) fmt.Fprintf(wtr, "AvgProvideDuration:\t%s\n", humanDuration(s.AvgProvideDuration)) fmt.Fprintf(wtr, "LastReprovideDuration:\t%s\n", humanDuration(s.LastReprovideDuration)) - fmt.Fprintf(wtr, "LastReprovideBatchSize:\t%s\n", humanNumber(s.LastReprovideBatchSize)) + if !s.LastRun.IsZero() { + fmt.Fprintf(wtr, "LastRun:\t%s\n", humanTime(s.LastRun)) + if s.fullRT { + fmt.Fprintf(wtr, "NextRun:\t%s\n", humanTime(s.LastRun.Add(s.ReprovideInterval))) + } + } return nil }), }, - Type: provider.ReproviderStats{}, + Type: reprovideStats{}, } func humanDuration(val time.Duration) string { return val.Truncate(time.Microsecond).String() } +func humanTime(val time.Time) string { + return val.Format("2006-01-02 15:04:05") +} + func humanNumber[T constraints.Float | constraints.Integer](n T) string { nf := float64(n) str := humanSI(nf, 0) diff --git a/docs/changelogs/v0.34.md b/docs/changelogs/v0.34.md index a131511a150..67c8d4c6ced 100644 --- a/docs/changelogs/v0.34.md +++ b/docs/changelogs/v0.34.md @@ -6,6 +6,8 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Reprovide command moved to routing](#reprovide-command-moved-to-routing) + - [Additional stats for Accelerated DHT Reprovides](#additional-stats-for-accelerated-dht-reprovides) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -13,6 +15,14 @@ ### ๐Ÿ”ฆ Highlights +#### Reprovide command moved to routing + +Moved the `bitswap reprovide` command to `routing reprovide`. ([#10677](https://github.com/ipfs/kubo/pull/10677)) + +#### Additional stats for Accelerated DHT Reprovides + +The `stats reprovide` command now shows additional stats for the DHT Accelerated Client, indicating the last and next `reprovide` times. ([#10677](https://github.com/ipfs/kubo/pull/10677)) + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 48ae0a5cf5f..7cf15ce5b35 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.23 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.27.2 + github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.38.2 github.com/multiformats/go-multiaddr v0.14.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 16a3a3dcc25..18c88b9339b 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -304,8 +304,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.27.2 h1:sGo4KdwBaMjdBjH08lqPJyt27Z4CO6sugne3ryX513s= -github.com/ipfs/boxo v0.27.2/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b h1:aHVcNBIAW/eKjnFpwABp+CgbhnYGBnEDdPLN1yYHQX0= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 35bee92bfa1..a35d4d88178 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.14 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 - github.com/ipfs/boxo v0.27.2 + github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 864cbcacd50..0e8614f3d8c 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.27.2 h1:sGo4KdwBaMjdBjH08lqPJyt27Z4CO6sugne3ryX513s= -github.com/ipfs/boxo v0.27.2/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b h1:aHVcNBIAW/eKjnFpwABp+CgbhnYGBnEDdPLN1yYHQX0= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/cli/delegated_routing_v1_http_proxy_test.go b/test/cli/delegated_routing_v1_http_proxy_test.go index ef3bc4fe2bc..7f8ff8bcaa8 100644 --- a/test/cli/delegated_routing_v1_http_proxy_test.go +++ b/test/cli/delegated_routing_v1_http_proxy_test.go @@ -72,7 +72,7 @@ func TestRoutingV1Proxy(t *testing.T) { cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000)) // Reprovide as initialProviderDelay still ongoing - res := nodes[0].IPFS("bitswap", "reprovide") + res := nodes[0].IPFS("routing", "reprovide") require.NoError(t, res.Err) res = nodes[1].IPFS("routing", "findprovs", cidStr) assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed()) diff --git a/test/cli/delegated_routing_v1_http_server_test.go b/test/cli/delegated_routing_v1_http_server_test.go index 916b9188280..8492e761c6c 100644 --- a/test/cli/delegated_routing_v1_http_server_test.go +++ b/test/cli/delegated_routing_v1_http_server_test.go @@ -40,7 +40,7 @@ func TestRoutingV1Server(t *testing.T) { cidStr := nodes[2].IPFSAddStr(text) _ = nodes[3].IPFSAddStr(text) // Reprovide as initialProviderDelay still ongoing - res := nodes[3].IPFS("bitswap", "reprovide") + res := nodes[3].IPFS("routing", "reprovide") require.NoError(t, res.Err) cid, err := cid.Decode(cidStr) diff --git a/test/cli/provider_test.go b/test/cli/provider_test.go index 546ac3fd711..81af7814954 100644 --- a/test/cli/provider_test.go +++ b/test/cli/provider_test.go @@ -43,7 +43,7 @@ func TestProvider(t *testing.T) { cid := nodes[0].IPFSAddStr(time.Now().String()) // Reprovide as initialProviderDelay still ongoing - res := nodes[0].IPFS("bitswap", "reprovide") + res := nodes[0].IPFS("routing", "reprovide") require.NoError(t, res.Err) expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) @@ -72,7 +72,7 @@ func TestProvider(t *testing.T) { expectNoProviders(t, cid, nodes[1:]...) - nodes[0].IPFS("bitswap", "reprovide") + nodes[0].IPFS("routing", "reprovide") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) @@ -89,7 +89,7 @@ func TestProvider(t *testing.T) { expectNoProviders(t, cid, nodes[1:]...) - nodes[0].IPFS("bitswap", "reprovide") + nodes[0].IPFS("routing", "reprovide") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) @@ -113,7 +113,7 @@ func TestProvider(t *testing.T) { expectNoProviders(t, cidBar, nodes[1:]...) expectNoProviders(t, cidBarDir, nodes[1:]...) - nodes[0].IPFS("bitswap", "reprovide") + nodes[0].IPFS("routing", "reprovide") expectNoProviders(t, cidFoo, nodes[1:]...) expectProviders(t, cidBar, nodes[0].PeerID().String(), nodes[1:]...) @@ -141,7 +141,7 @@ func TestProvider(t *testing.T) { expectNoProviders(t, cidBar, nodes[1:]...) expectNoProviders(t, cidBarDir, nodes[1:]...) - nodes[0].IPFS("bitswap", "reprovide") + nodes[0].IPFS("routing", "reprovide") expectNoProviders(t, cidFoo, nodes[1:]...) expectNoProviders(t, cidBar, nodes[1:]...) @@ -161,7 +161,7 @@ func TestProvider(t *testing.T) { expectNoProviders(t, cid, nodes[1:]...) - nodes[0].IPFS("bitswap", "reprovide") + nodes[0].IPFS("routing", "reprovide") expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) diff --git a/test/cli/routing_dht_test.go b/test/cli/routing_dht_test.go index d149e93a2a3..9322d8cc164 100644 --- a/test/cli/routing_dht_test.go +++ b/test/cli/routing_dht_test.go @@ -85,7 +85,7 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) { t.Parallel() hash := nodes[3].IPFSAddStr("some stuff") // Reprovide as initialProviderDelay still ongoing - res := nodes[3].IPFS("bitswap", "reprovide") + res := nodes[3].IPFS("routing", "reprovide") require.NoError(t, res.Err) res = nodes[4].IPFS("routing", "findprovs", hash) assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed()) diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 49ba2f1c99f..f6017ccfd37 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -120,7 +120,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.27.2 // indirect + github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 49fd0c4ce82..a82a70b0456 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -324,8 +324,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.27.2 h1:sGo4KdwBaMjdBjH08lqPJyt27Z4CO6sugne3ryX513s= -github.com/ipfs/boxo v0.27.2/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b h1:aHVcNBIAW/eKjnFpwABp+CgbhnYGBnEDdPLN1yYHQX0= +github.com/ipfs/boxo v0.27.3-0.20250131141414-5c158ecc3b0b/go.mod h1:qEIRrGNr0bitDedTCzyzBHxzNWqYmyuHgK8LG9Q83EM= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=