gateway: implement IPIP-0445 (skip-raw-blocks option)

gateway/blocks_backend.go

@@ -296,6 +296,10 @@ func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (Con
 var emptyRoot = []cid.Cid{cid.MustParse("bafkqaaa")}
 
 func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
+	if params.SkipRawBlocks.Bool() && p.RootCid().Prefix().Codec == cid.Raw {
+
+	}
+
 	pathMetadata, err := bb.ResolvePath(ctx, p)
 	if err != nil {
 		rootCid, err := cid.Decode(strings.Split(p.String(), "/")[2])
@@ -312,8 +316,9 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param
 		blockGetter := merkledag.NewDAGService(bb.blockService).Session(ctx)
 
 		blockGetter = &nodeGetterToCarExporer{
-			ng: blockGetter,
-			cw: cw,
+			ng:            blockGetter,
+			cw:            cw,
+			skipRawBlocks: params.SkipRawBlocks.Bool(),
 		}
 
 		// Setup the UnixFS resolver.
@@ -352,8 +357,9 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param
 		blockGetter := merkledag.NewDAGService(bb.blockService).Session(ctx)
 
 		blockGetter = &nodeGetterToCarExporer{
-			ng: blockGetter,
-			cw: cw,
+			ng:            blockGetter,
+			cw:            cw,
+			skipRawBlocks: params.SkipRawBlocks.Bool(),
 		}
 
 		// Setup the UnixFS resolver.
@@ -732,8 +738,9 @@ func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.Imm
 }
 
 type nodeGetterToCarExporer struct {
-	ng format.NodeGetter
-	cw storage.WritableCar
+	ng            format.NodeGetter
+	cw            storage.WritableCar
+	skipRawBlocks bool
 }
 
 func (n *nodeGetterToCarExporer) Get(ctx context.Context, c cid.Cid) (format.Node, error) {
@@ -774,6 +781,19 @@ func (n *nodeGetterToCarExporer) GetMany(ctx context.Context, cids []cid.Cid) <-
 }
 
 func (n *nodeGetterToCarExporer) trySendBlock(ctx context.Context, block blocks.Block) error {
+	// FIXME(@Jorropo): this is very inneficient, we fetch all blocks even if we don't send them.
+	// I've tried doing so using the ipld stack however the problem is that filtering on the
+	// selector or traversal callback does not work because the unixfs reifier is ran before,
+	// so trying to filter raw links do nothing because go-unixfsnode removed them already,
+	// so we need to filter in a callback from unixfsnode but the reifier does not know a about
+	// [traversal.SkipMe] making it a lost cause. I've looked into updating unixfsnode but this
+	// much more work because there are no easy way to pass options or understand what the side
+	// effects of this would be.
+	// Abstractions everywhere yet a simple small behaviour change require rethinking everything :'(.
+	// Will fix with boxo/unixfs.
+	if n.skipRawBlocks && block.Cid().Prefix().Codec == cid.Raw {
+		return nil
+	}
 	return n.cw.Put(ctx, block.Cid().KeyString(), block.RawData())
 }
 

gateway/gateway.go

@@ -95,10 +95,11 @@ type PublicGateway struct {
 }
 
 type CarParams struct {
-	Range      *DagByteRange
-	Scope      DagScope
-	Order      DagOrder
-	Duplicates DuplicateBlocksPolicy
+	Range         *DagByteRange
+	Scope         DagScope
+	Order         DagOrder
+	Duplicates    DuplicateBlocksPolicy
+	SkipRawBlocks SkipRawBlocksPolicy
 }
 
 // DagByteRange describes a range request within a UnixFS file. "From" and
@@ -211,6 +212,46 @@ func (d DuplicateBlocksPolicy) String() string {
 	return ""
 }
 
+// SkipRawBlocksPolicy represents the get parameter 'skip-raw-blocks' (IPIP-445)
+type SkipRawBlocksPolicy uint8
+
+const (
+	SkipRawBlocksImplicit SkipRawBlocksPolicy = iota // implicit default and not skip leaves
+	SendRawBlocks                                    // explicitely do not skip leaves
+	SkipRawBlocks                                    // explicitly skip leaves
+)
+
+func NewSkipRawBlocksPolicy(v string) (SkipRawBlocksPolicy, error) {
+	switch v {
+	case "y":
+		return SkipRawBlocks, nil
+	case "n":
+		return SendRawBlocks, nil
+	case "":
+		return SkipRawBlocksImplicit, nil
+	}
+	return 0, fmt.Errorf("unsupported skip-raw-blocks GET parameter: %q", v)
+}
+
+func (d SkipRawBlocksPolicy) Bool() bool {
+	// duplicates should be returned only when explicitly requested,
+	// so any other state than SkipRawBlocksIncluded should return false
+	return d == SkipRawBlocks
+}
+
+func (d SkipRawBlocksPolicy) String() string {
+	switch d {
+	case SkipRawBlocksImplicit:
+		return ""
+	case SkipRawBlocks:
+		return "y"
+	case SendRawBlocks:
+		return "n"
+	default:
+		return strconv.FormatUint(uint64(d), 10)
+	}
+}
+
 type ContentPathMetadata struct {
 	PathSegmentRoots     []cid.Cid
 	LastSegment          path.ImmutablePath

gateway/handler_car.go

@@ -22,6 +22,7 @@ import (
 const (
 	carRangeBytesKey          = "entity-bytes"
 	carTerminalElementTypeKey = "dag-scope"
+	carSkipRawBlocksTypeKey   = "skip-raw-blocks"
 )
 
 // serveCAR returns a CAR stream for specific DAG+selector
@@ -118,6 +119,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 	queryParams := r.URL.Query()
 	rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey)
 	scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey)
+	skipRawBlocksStr, hasSkipRawBlocks := queryParams.Get(carSkipRawBlocksTypeKey), queryParams.Has(carSkipRawBlocksTypeKey)
 
 	params := CarParams{}
 	if hasRange {
@@ -141,6 +143,15 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 		params.Scope = DagScopeAll
 	}
 
+	if hasSkipRawBlocks {
+		// skip leaves from IPIP-445
+		skip, err := NewSkipRawBlocksPolicy(skipRawBlocksStr)
+		if err != nil {
+			return CarParams{}, err
+		}
+		params.SkipRawBlocks = skip
+	}
+
 	// application/vnd.ipld.car content type parameters from Accept header
 
 	// version of CAR format
@@ -249,6 +260,11 @@ func getCarEtag(imPath path.ImmutablePath, params CarParams, rootCid cid.Cid) st
 		h.WriteString("\x00dups=y")
 	}
 
+	// 'skip-leaves' from IPIP-445 impact Etag only if 'y'
+	if skip := params.SkipRawBlocks; skip == SkipRawBlocks {
+		h.WriteString("\x00skip-leaves=y")
+	}
+
 	if params.Range != nil {
 		if params.Range.From != 0 || params.Range.To != nil {
 			h.WriteString("\x00range=")