From f7bb4b3e52b57559295ca94eeda2445f9e8cf7fc Mon Sep 17 00:00:00 2001 From: Jorropo Date: Sun, 15 Oct 2023 11:10:54 +0200 Subject: [PATCH] gateway: implement ipip-0445 See https://github.com/ipfs/specs/pull/445 --- gateway/blocks_backend.go | 32 ++++++++++++++++++++----- gateway/gateway.go | 49 +++++++++++++++++++++++++++++++++++---- gateway/handler_car.go | 16 +++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index b4dd705d2..781874d01 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -296,6 +296,10 @@ func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (Con var emptyRoot = []cid.Cid{cid.MustParse("bafkqaaa")} func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { + if params.SkipRawBlocks.Bool() && p.RootCid().Prefix().Codec == cid.Raw { + + } + pathMetadata, err := bb.ResolvePath(ctx, p) if err != nil { rootCid, err := cid.Decode(strings.Split(p.String(), "/")[2]) @@ -312,8 +316,9 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param blockGetter := merkledag.NewDAGService(bb.blockService).Session(ctx) blockGetter = &nodeGetterToCarExporer{ - ng: blockGetter, - cw: cw, + ng: blockGetter, + cw: cw, + skipRawBlocks: params.SkipRawBlocks.Bool(), } // Setup the UnixFS resolver. @@ -352,8 +357,9 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, param blockGetter := merkledag.NewDAGService(bb.blockService).Session(ctx) blockGetter = &nodeGetterToCarExporer{ - ng: blockGetter, - cw: cw, + ng: blockGetter, + cw: cw, + skipRawBlocks: params.SkipRawBlocks.Bool(), } // Setup the UnixFS resolver. @@ -732,8 +738,9 @@ func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.Imm } type nodeGetterToCarExporer struct { - ng format.NodeGetter - cw storage.WritableCar + ng format.NodeGetter + cw storage.WritableCar + skipRawBlocks bool } func (n *nodeGetterToCarExporer) Get(ctx context.Context, c cid.Cid) (format.Node, error) { @@ -774,6 +781,19 @@ func (n *nodeGetterToCarExporer) GetMany(ctx context.Context, cids []cid.Cid) <- } func (n *nodeGetterToCarExporer) trySendBlock(ctx context.Context, block blocks.Block) error { + // FIXME(@Jorropo): this is very inneficient, we fetch all blocks even if we don't send them. + // I've tried doing so using the ipld stack however the problem is that filtering on the + // selector or traversal callback does not work because the unixfs reifier is ran before, + // so trying to filter raw links do nothing because go-unixfsnode removed them already, + // so we need to filter in a callback from unixfsnode but the reifier does not know a about + // [traversal.SkipMe] making it a lost cause. I've looked into updating unixfsnode but this + // much more work because there are no easy way to pass options or understand what the side + // effects of this would be. + // Abstractions everywhere yet a simple small behaviour change require rethinking everything :'(. + // Will fix with boxo/unixfs. + if n.skipRawBlocks && block.Cid().Prefix().Codec == cid.Raw { + return nil + } return n.cw.Put(ctx, block.Cid().KeyString(), block.RawData()) } diff --git a/gateway/gateway.go b/gateway/gateway.go index 28cf5d295..da3649950 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -95,10 +95,11 @@ type PublicGateway struct { } type CarParams struct { - Range *DagByteRange - Scope DagScope - Order DagOrder - Duplicates DuplicateBlocksPolicy + Range *DagByteRange + Scope DagScope + Order DagOrder + Duplicates DuplicateBlocksPolicy + SkipRawBlocks SkipRawBlocksPolicy } // DagByteRange describes a range request within a UnixFS file. "From" and @@ -211,6 +212,46 @@ func (d DuplicateBlocksPolicy) String() string { return "" } +// SkipRawBlocksPolicy represents the get parameter 'skip-raw-blocks' (IPIP-445) +type SkipRawBlocksPolicy uint8 + +const ( + SkipRawBlocksImplicit SkipRawBlocksPolicy = iota // implicit default and not skip leaves + SendRawBlocks // explicitely do not skip leaves + SkipRawBlocks // explicitly skip leaves +) + +func NewSkipRawBlocksPolicy(v string) (SkipRawBlocksPolicy, error) { + switch v { + case "y": + return SkipRawBlocks, nil + case "n": + return SendRawBlocks, nil + case "": + return SkipRawBlocksImplicit, nil + } + return 0, fmt.Errorf("unsupported skip-raw-blocks GET parameter: %q", v) +} + +func (d SkipRawBlocksPolicy) Bool() bool { + // duplicates should be returned only when explicitly requested, + // so any other state than SkipRawBlocksIncluded should return false + return d == SkipRawBlocks +} + +func (d SkipRawBlocksPolicy) String() string { + switch d { + case SkipRawBlocksImplicit: + return "" + case SkipRawBlocks: + return "y" + case SendRawBlocks: + return "n" + default: + return strconv.FormatUint(uint64(d), 10) + } +} + type ContentPathMetadata struct { PathSegmentRoots []cid.Cid LastSegment path.ImmutablePath diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 21b90108c..e47e296e3 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -22,6 +22,7 @@ import ( const ( carRangeBytesKey = "entity-bytes" carTerminalElementTypeKey = "dag-scope" + carSkipRawBlocksTypeKey = "skip-raw-blocks" ) // serveCAR returns a CAR stream for specific DAG+selector @@ -118,6 +119,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa queryParams := r.URL.Query() rangeStr, hasRange := queryParams.Get(carRangeBytesKey), queryParams.Has(carRangeBytesKey) scopeStr, hasScope := queryParams.Get(carTerminalElementTypeKey), queryParams.Has(carTerminalElementTypeKey) + skipRawBlocksStr, hasSkipRawBlocks := queryParams.Get(carSkipRawBlocksTypeKey), queryParams.Has(carSkipRawBlocksTypeKey) params := CarParams{} if hasRange { @@ -141,6 +143,15 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa params.Scope = DagScopeAll } + if hasSkipRawBlocks { + // skip leaves from IPIP-445 + skip, err := NewSkipRawBlocksPolicy(skipRawBlocksStr) + if err != nil { + return CarParams{}, err + } + params.SkipRawBlocks = skip + } + // application/vnd.ipld.car content type parameters from Accept header // version of CAR format @@ -249,6 +260,11 @@ func getCarEtag(imPath path.ImmutablePath, params CarParams, rootCid cid.Cid) st h.WriteString("\x00dups=y") } + // 'skip-leaves' from IPIP-445 impact Etag only if 'y' + if skip := params.SkipRawBlocks; skip == SkipRawBlocks { + h.WriteString("\x00skip-leaves=y") + } + if params.Range != nil { if params.Range.From != 0 || params.Range.To != nil { h.WriteString("\x00range=")