Merge branch 'master' into chore/migrate-peering-to-boxo

ipfs · Oct 31, 2023 · 151da63 · 151da63
2 parents 55304ba + 33bbee5
commit 151da63
Show file tree

Hide file tree

Showing 19 changed files with 596 additions and 75 deletions.
diff --git a/README.md b/README.md
@@ -30,6 +30,7 @@ Featureset
 - [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon
 - [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API
 - [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node
+- [Content blocking](/docs/content-blocking.md) support for operators of public nodes
 
 ### Other implementations
 
@@ -88,7 +89,6 @@ Before opening an issue, consider using one of the following locations to ensure
     - [Install Go](#install-go)
     - [Download and Compile IPFS](#download-and-compile-ipfs)
       - [Cross Compiling](#cross-compiling)
-      - [OpenSSL](#openssl)
     - [Troubleshooting](#troubleshooting)
 - [Getting Started](#getting-started)
   - [Usage](#usage)
@@ -327,15 +327,6 @@ Compiling for a different platform is as simple as running:
 make build GOOS=myTargetOS GOARCH=myTargetArchitecture
 ```
 
-##### OpenSSL
-
-To build go-ipfs with OpenSSL support, append `GOTAGS=openssl` to your `make` invocation. Building with OpenSSL should significantly reduce the background CPU usage on nodes that frequently make or receive new connections.
-
-Note: OpenSSL requires CGO support and, by default, CGO is disabled when cross-compiling. To cross-compile with OpenSSL support, you must:
-
-1. Install a compiler toolchain for the target platform.
-2. Set the `CGO_ENABLED=1` environment variable.
-
 #### Troubleshooting
 
 - Separate [instructions are available for building on Windows](docs/windows.md).

diff --git a/core/commands/dag/export.go b/core/commands/dag/export.go
@@ -14,26 +14,32 @@ import (
 	cid "github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/kubo/core/commands/cmdenv"
+	"github.com/ipfs/kubo/core/commands/cmdutils"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	gocar "github.com/ipld/go-car"
 	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 )
 
 func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
-	c, err := cid.Decode(req.Arguments[0])
+	// Accept CID or a content path
+	p, err := cmdutils.PathOrCidPath(req.Arguments[0])
 	if err != nil {
-		return fmt.Errorf(
-			"unable to parse root specification (currently only bare CIDs are supported): %s",
-			err,
-		)
+		return err
 	}
 
 	api, err := cmdenv.GetApi(env, req)
 	if err != nil {
 		return err
 	}
 
+	// Resolve path and confirm the root block is available, fail fast if not
+	b, err := api.Block().Stat(req.Context, p)
+	if err != nil {
+		return err
+	}
+	c := b.Path().RootCid()
+
 	pipeR, pipeW := io.Pipe()
 
 	errCh := make(chan error, 2) // we only report the 1st error

diff --git a/core/core.go b/core/core.go
@@ -76,35 +76,39 @@ type IpfsNode struct {
 	PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
 
 	// Services
-	Peerstore            pstore.Peerstore          `optional:"true"` // storage for other Peer instances
-	Blockstore           bstore.GCBlockstore       // the block store (lower level)
-	Filestore            *filestore.Filestore      `optional:"true"` // the filestore blockstore
-	BaseBlocks           node.BaseBlocks           // the raw blockstore, no filestore wrapping
-	GCLocker             bstore.GCLocker           // the locker used to protect the blockstore during gc
-	Blocks               bserv.BlockService        // the block service, get/add blocks.
-	DAG                  ipld.DAGService           // the merkle dag service, get/add objects.
-	IPLDFetcherFactory   fetcher.Factory           `name:"ipldFetcher"`   // fetcher that paths over the IPLD data model
-	UnixFSFetcherFactory fetcher.Factory           `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
-	Reporter             *metrics.BandwidthCounter `optional:"true"`
-	Discovery            mdns.Service              `optional:"true"`
-	FilesRoot            *mfs.Root
-	RecordValidator      record.Validator
+	Peerstore                   pstore.Peerstore          `optional:"true"` // storage for other Peer instances
+	Blockstore                  bstore.GCBlockstore       // the block store (lower level)
+	Filestore                   *filestore.Filestore      `optional:"true"` // the filestore blockstore
+	BaseBlocks                  node.BaseBlocks           // the raw blockstore, no filestore wrapping
+	GCLocker                    bstore.GCLocker           // the locker used to protect the blockstore during gc
+	Blocks                      bserv.BlockService        // the block service, get/add blocks.
+	DAG                         ipld.DAGService           // the merkle dag service, get/add objects.
+	IPLDFetcherFactory          fetcher.Factory           `name:"ipldFetcher"`          // fetcher that paths over the IPLD data model
+	UnixFSFetcherFactory        fetcher.Factory           `name:"unixfsFetcher"`        // fetcher that interprets UnixFS data
+	OfflineIPLDFetcherFactory   fetcher.Factory           `name:"offlineIpldFetcher"`   // fetcher that paths over the IPLD data model without fetching new blocks
+	OfflineUnixFSFetcherFactory fetcher.Factory           `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks
+	Reporter                    *metrics.BandwidthCounter `optional:"true"`
+	Discovery                   mdns.Service              `optional:"true"`
+	FilesRoot                   *mfs.Root
+	RecordValidator             record.Validator
 
 	// Online
-	PeerHost           p2phost.Host               `optional:"true"` // the network host (server+client)
-	Peering            *peering.PeeringService    `optional:"true"`
-	Filters            *ma.Filters                `optional:"true"`
-	Bootstrapper       io.Closer                  `optional:"true"` // the periodic bootstrapper
-	Routing            irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
-	DNSResolver        *madns.Resolver            // the DNS resolver
-	IPLDPathResolver   pathresolver.Resolver      `name:"ipldPathResolver"`   // The IPLD path resolver
-	UnixFSPathResolver pathresolver.Resolver      `name:"unixFSPathResolver"` // The UnixFS path resolver
-	Exchange           exchange.Interface         // the block exchange + strategy (bitswap)
-	Namesys            namesys.NameSystem         // the name system, resolves paths to hashes
-	Provider           provider.System            // the value provider system
-	IpnsRepub          *ipnsrp.Republisher        `optional:"true"`
-	GraphExchange      graphsync.GraphExchange    `optional:"true"`
-	ResourceManager    network.ResourceManager    `optional:"true"`
+	PeerHost                  p2phost.Host               `optional:"true"` // the network host (server+client)
+	Peering                   *peering.PeeringService    `optional:"true"`
+	Filters                   *ma.Filters                `optional:"true"`
+	Bootstrapper              io.Closer                  `optional:"true"` // the periodic bootstrapper
+	Routing                   irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
+	DNSResolver               *madns.Resolver            // the DNS resolver
+	IPLDPathResolver          pathresolver.Resolver      `name:"ipldPathResolver"`          // The IPLD path resolver
+	UnixFSPathResolver        pathresolver.Resolver      `name:"unixFSPathResolver"`        // The UnixFS path resolver
+	OfflineIPLDPathResolver   pathresolver.Resolver      `name:"offlineIpldPathResolver"`   // The IPLD path resolver that uses only locally available blocks
+	OfflineUnixFSPathResolver pathresolver.Resolver      `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
+	Exchange                  exchange.Interface         // the block exchange + strategy (bitswap)
+	Namesys                   namesys.NameSystem         // the name system, resolves paths to hashes
+	Provider                  provider.System            // the value provider system
+	IpnsRepub                 *ipnsrp.Republisher        `optional:"true"`
+	GraphExchange             graphsync.GraphExchange    `optional:"true"`
+	ResourceManager           network.ResourceManager    `optional:"true"`
 
 	PubSub   *pubsub.PubSub             `optional:"true"`
 	PSRouter *psrouter.PubsubValueStore `optional:"true"`

diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go
@@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
 
-		backend, err := gateway.NewBlocksBackend(bserv)
+		backend, err := gateway.NewBlocksBackend(bserv,
+			// GatewayOverLibp2p only returns things that are in local blockstore
+			// (same as Gateway.NoFetch=true), we have to pass offline path resolver
+			gateway.WithResolver(n.OfflineUnixFSPathResolver),
+		)
 		if err != nil {
 			return nil, err
 		}
@@ -111,6 +115,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
 	bserv := n.Blocks
 	var vsRouting routing.ValueStore = n.Routing
 	nsys := n.Namesys
+	pathResolver := n.UnixFSPathResolver
+
 	if cfg.Gateway.NoFetch {
 		bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
 
@@ -130,9 +136,17 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
 		if err != nil {
 			return nil, fmt.Errorf("error constructing namesys: %w", err)
 		}
+
+		// Gateway.NoFetch=true requires offline path resolver
+		// to avoid fetching missing blocks during path traversal
+		pathResolver = n.OfflineUnixFSPathResolver
 	}
 
-	backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
+	backend, err := gateway.NewBlocksBackend(bserv,
+		gateway.WithValueStore(vsRouting),
+		gateway.WithNameSystem(nsys),
+		gateway.WithResolver(pathResolver),
+	)
 	if err != nil {
 		return nil, err
 	}

diff --git a/core/node/core.go b/core/node/core.go
@@ -7,6 +7,7 @@ import (
 	"github.com/ipfs/boxo/blockservice"
 	blockstore "github.com/ipfs/boxo/blockstore"
 	exchange "github.com/ipfs/boxo/exchange"
+	offline "github.com/ipfs/boxo/exchange/offline"
 	"github.com/ipfs/boxo/fetcher"
 	bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
 	"github.com/ipfs/boxo/filestore"
@@ -21,9 +22,6 @@ import (
 	format "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-unixfsnode"
 	dagpb "github.com/ipld/go-codec-dagpb"
-	"github.com/ipld/go-ipld-prime"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/schema"
 	"go.uber.org/fx"
 
 	"github.com/ipfs/kubo/core/node/helpers"
@@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter {
 // FetchersOut allows injection of fetchers.
 type FetchersOut struct {
 	fx.Out
-	IPLDFetcher   fetcher.Factory `name:"ipldFetcher"`
-	UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+	IPLDFetcher          fetcher.Factory `name:"ipldFetcher"`
+	UnixfsFetcher        fetcher.Factory `name:"unixfsFetcher"`
+	OfflineIPLDFetcher   fetcher.Factory `name:"offlineIpldFetcher"`
+	OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
 }
 
 // FetchersIn allows using fetchers for other dependencies.
 type FetchersIn struct {
 	fx.In
-	IPLDFetcher   fetcher.Factory `name:"ipldFetcher"`
-	UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
+	IPLDFetcher          fetcher.Factory `name:"ipldFetcher"`
+	UnixfsFetcher        fetcher.Factory `name:"unixfsFetcher"`
+	OfflineIPLDFetcher   fetcher.Factory `name:"offlineIpldFetcher"`
+	OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
 }
 
 // FetcherConfig returns a fetcher config that can build new fetcher instances
 func FetcherConfig(bs blockservice.BlockService) FetchersOut {
 	ipldFetcher := bsfetcher.NewFetcherConfig(bs)
-	ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
-		if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
-			return tlnkNd.LinkTargetNodePrototype(), nil
-		}
-		return basicnode.Prototype.Any, nil
-	})
-
+	ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
 	unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
-	return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher}
+
+	// Construct offline versions which we can safely use in contexts where
+	// path resolution should not fetch new blocks via exchange.
+	offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
+	offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
+	offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
+	offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
+
+	return FetchersOut{
+		IPLDFetcher:          ipldFetcher,
+		UnixfsFetcher:        unixFSFetcher,
+		OfflineIPLDFetcher:   offlineIpldFetcher,
+		OfflineUnixfsFetcher: offlineUnixFSFetcher,
+	}
 }
 
 // PathResolversOut allows injection of path resolvers
 type PathResolversOut struct {
 	fx.Out
-	IPLDPathResolver   pathresolver.Resolver `name:"ipldPathResolver"`
-	UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
+	IPLDPathResolver          pathresolver.Resolver `name:"ipldPathResolver"`
+	UnixFSPathResolver        pathresolver.Resolver `name:"unixFSPathResolver"`
+	OfflineIPLDPathResolver   pathresolver.Resolver `name:"offlineIpldPathResolver"`
+	OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"`
 }
 
 // PathResolverConfig creates path resolvers with the given fetchers.
 func PathResolverConfig(fetchers FetchersIn) PathResolversOut {
 	return PathResolversOut{
-		IPLDPathResolver:   pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
-		UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
+		IPLDPathResolver:          pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
+		UnixFSPathResolver:        pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
+		OfflineIPLDPathResolver:   pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher),
+		OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher),
 	}
 }
 

diff --git a/docs/changelogs/v0.24.md b/docs/changelogs/v0.24.md
@@ -6,6 +6,7 @@
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+  - [Support for content blocking](#support-for-content-blocking)
   - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful)
   - [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults)
   - [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching)
@@ -16,6 +17,14 @@
 
 ### 🔦 Highlights
 
+#### Support for content blocking
+
+This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/).
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+The official build does not ship with any denylists.
+
+Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md)
+
 #### Gateway: the root of the CARs are no longer meaningful
 
 When requesting a CAR from the gateway, the root of the CAR might no longer be

diff --git a/docs/content-blocking.md b/docs/content-blocking.md
@@ -0,0 +1,73 @@
+<h1 align="center">
+  <br>
+  <a href="#readme"><img src="https://github.com/ipfs-shipyard/nopfs/blob/41484a818e6542314f784da852fc41b76f2d48a6/logo.png?raw=true" alt="content blocking logo" title="content blocking in Kubo" width="200"></a>
+  <br>
+  Content Blocking in Kubo
+  <br>
+</h1>
+
+Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
+
+## Default behavior
+
+Official Kubo build does not ship with any denylists enabled by default.
+
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+
+## How to enable blocking
+
+Place a `*.deny` file in one of directories:
+
+- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set)
+- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not  set)
+- `/etc/ipfs/denylists/` (global)
+
+Files need to be present before starting the `ipfs daemon` in order to be watched for updates.
+
+If a new denylist file is added, `ipfs daemon` needs to be restarted.
+
+CLI and Gateway users will receive errors in response to request impacted by a blocklist:
+
+```
+Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided
+```
+
+End user is not informed about the exact reason, see [How to
+debug](#how-to-debug) if you need to find out which line of which denylist
+caused the request to be blocked.
+
+## Denylist file format
+
+[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
+
+Clear-text rules are simple: just put content paths to block, one per line.
+Paths with unicode and whitespace need to be percend-encoded:
+
+```
+/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png
+```
+
+Sensitive content paths can be double-hashed to block without revealing them.
+Double-hashed list example: https://badbits.dwebops.pub/badbits.deny
+
+See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples.
+
+## How to suspend blocking without removing denylists
+
+Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon.
+
+
+## How to debug
+
+Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`
+
+All block events are logged as warnings on a separate level named `nopfs-blocks`.
+
+To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`:
+
+```
+WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9)
+```
+
+
diff --git a/docs/environment-variables.md b/docs/environment-variables.md
@@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for
 inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
 or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
 
+
+## `IPFS_CONTENT_BLOCKING_DISABLE`
+
+Disables the content-blocking subsystem. No denylists will be watched and no
+content will be blocked.
+
 ## `LIBP2P_TCP_REUSEPORT`
 
 Kubo tries to reuse the same source port for all connections to improve NAT