Merge pull request #613 from ipfs/release-v0.20.0

Release v0.20.0

.github/workflows/gateway-conformance.yml

@@ -1,17 +1,23 @@
 name: Gateway Conformance
+# This workflow runs https://github.com/ipfs/gateway-conformance
+# against different backend implementations of boxo/gateway
 
 on:
   push:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  gateway-conformance:
+  # This test uses a static CAR file as a local blockstore,
+  # allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go)
+  # which is used by implementations like Kubo
+  local-block-backend:
     runs-on: ubuntu-latest
     steps:
       # 1. Download the gateway-conformance fixtures
@@ -21,49 +27,185 @@ jobs:
           output: fixtures
           merged: true
 
-      # 2. Build the car-gateway
+      # 2. Build the gateway binary
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build test-gateway
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/car-file
+
+      # 3. Start the gateway binary
+      - name: Start test-gateway
+        run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/[email protected]
+        with:
+          gateway-url: http://127.0.0.1:8040
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          subdomain-url: http://example.net
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.json
+          path: output.json
+
+  # This test uses remote block gateway (?format=raw) as a remote blockstore,
+  # allowing us to test conformance against RemoteBlocksBackend
+  # (gateway/backend_blocks.go) which is used by implementations like
+  # rainbow configured to use with remote block backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
+  remote-block-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/[email protected]
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-block-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-block-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-blocks
+
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.raw
+        run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests
+      - name: Start gateway that uses the remote block backend
+        run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/[email protected]
+        with:
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          subdomain-url: http://example.net
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
         with:
-          go-version: 1.21.x
+          name: gateway-conformance_remote-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_remote-block-backend.json
+          path: output.json
+
+  # This test uses remote CAR gateway (?format=car, IPIP-402)
+  # allowing us to test conformance against remote CarFetcher backend.
+  # (gateway/backend_car_fetcher.go) which is used by implementations like
+  # rainbow configured to use with remote car backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car
+  remote-car-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/[email protected]
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
       - name: Checkout boxo
         uses: actions/checkout@v4
         with:
           path: boxo
-      - name: Build car-gateway
-        run: go build -o car-gateway
-        working-directory: boxo/examples/gateway/car
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-car-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-car-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-car
 
-      # 3. Start the car-gateway
-      - name: Start car-gateway
-        run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 &
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402)
+        run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests
+      - name: Start gateway that uses the remote CAR backend
+        run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 &
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
         uses: ipfs/gateway-conformance/.github/actions/[email protected]
         with:
-          gateway-url: http://127.0.0.1:8040
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
           json: output.json
           xml: output.xml
           html: output.html
           markdown: output.md
           subdomain-url: http://example.net
           specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
-          args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
 
       # 5. Upload the results
       - name: Upload MD summary
         if: failure() || success()
         run: cat output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload HTML report
         if: failure() || success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.html
+          name: gateway-conformance_remote-car-backend.html
           path: output.html
       - name: Upload JSON report
         if: failure() || success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.json
+          name: gateway-conformance_remote-car-backend.json
           path: output.json

.github/workflows/gateway-sharness.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.22.x
       - name: Checkout boxo
         uses: actions/checkout@v3
         with:

CHANGELOG.md

@@ -24,6 +24,22 @@ The following emojis are used to highlight certain changes:
 
 ### Security
 
+## [v0.20.0]
+
+### Added
+
+* ✨ `gateway` has new backend possibilities:
+  * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
+  * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
+* `gateway` now sets the [`Content-Location`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Location) header for requests with non-default content format, as a result of content negotiation. This allows generic and misconfigured HTTP caches to store Deserialized, CAR and Block responses separately, under distinct cache keys.
+* `gateway` now supports `car-dups`, `car-order` and `car-version` as query parameters in addition to the `application/vnd.ipld.car` parameters sent via `Accept` header. The parameters in the `Accept` header have always priority, but including them in URL simplifies HTTP caching and allows use in `Content-Location` header on CAR responses to maximize interoperability with wide array of HTTP caches.
+* `bitswap/server` now allows to override the default peer ledger with `WithPeerLedger`.
+
+### Fixed
+
+* `routing/http/server` now returns 404 Status Not Found when no records can be found.
+* `routing/http/server` now supports legacy RSA PeerIDs encoded as Base58 Multihash
+
 ## [v0.19.0]
 
 ### Added

bitswap/network/ipfs_impl.go

@@ -11,17 +11,17 @@ import (
 	bsmsg "github.com/ipfs/boxo/bitswap/message"
 	"github.com/ipfs/boxo/bitswap/network/internal"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p/core/connmgr"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
-	peerstore "github.com/libp2p/go-libp2p/core/peerstore"
+	"github.com/libp2p/go-libp2p/core/peerstore"
 	"github.com/libp2p/go-libp2p/core/protocol"
 	"github.com/libp2p/go-libp2p/core/routing"
 	"github.com/libp2p/go-libp2p/p2p/protocol/ping"
-	msgio "github.com/libp2p/go-msgio"
+	"github.com/libp2p/go-msgio"
 	ma "github.com/multiformats/go-multiaddr"
 	"github.com/multiformats/go-multistream"
 )
@@ -452,7 +452,7 @@ func (nn *netNotifiee) impl() *impl {
 
 func (nn *netNotifiee) Connected(n network.Network, v network.Conn) {
 	// ignore transient connections
-	if v.Stat().Transient {
+	if v.Stat().Limited {
 		return
 	}
 

bitswap/options.go

@@ -59,6 +59,10 @@ func WithScoreLedger(scoreLedger server.ScoreLedger) Option {
 	return Option{server.WithScoreLedger(scoreLedger)}
 }
 
+func WithPeerLedger(peerLedger server.PeerLedger) Option {
+	return Option{server.WithPeerLedger(peerLedger)}
+}
+
 func WithTargetMessageSize(tms int) Option {
 	return Option{server.WithTargetMessageSize(tms)}
 }

bitswap/server/forward.go

@@ -11,4 +11,6 @@ type (
 	TaskInfo               = decision.TaskInfo
 	ScoreLedger            = decision.ScoreLedger
 	ScorePeerFunc          = decision.ScorePeerFunc
+	PeerLedger             = decision.PeerLedger
+	PeerEntry              = decision.PeerEntry
 )

bitswap/server/internal/decision/engine.go

@@ -123,6 +123,42 @@ type ScoreLedger interface {
 	Stop()
 }
 
+type PeerEntry struct {
+	Peer     peer.ID
+	Priority int32
+	WantType pb.Message_Wantlist_WantType
+}
+
+// PeerLedger is an external ledger dealing with peers and their want lists.
+type PeerLedger interface {
+	// Wants informs the ledger that [peer.ID] wants [wl.Entry].
+	Wants(p peer.ID, e wl.Entry)
+
+	// CancelWant returns true if the [cid.Cid] is present in the wantlist of [peer.ID].
+	CancelWant(p peer.ID, k cid.Cid) bool
+
+	// CancelWantWithType will not cancel WantBlock if we sent a HAVE message.
+	CancelWantWithType(p peer.ID, k cid.Cid, typ pb.Message_Wantlist_WantType)
+
+	// Peers returns all peers that want [cid.Cid].
+	Peers(k cid.Cid) []PeerEntry
+
+	// CollectPeerIDs returns all peers that the ledger has an active session with.
+	CollectPeerIDs() []peer.ID
+
+	// WantlistSizeForPeer returns the size of the wantlist for [peer.ID].
+	WantlistSizeForPeer(p peer.ID) int
+
+	// WantlistForPeer returns the wantlist for [peer.ID].
+	WantlistForPeer(p peer.ID) []wl.Entry
+
+	// ClearPeerWantlist clears the wantlist for [peer.ID].
+	ClearPeerWantlist(p peer.ID)
+
+	// PeerDisconnected informs the ledger that [peer.ID] is no longer connected.
+	PeerDisconnected(p peer.ID)
+}
+
 // Engine manages sending requested blocks to peers.
 type Engine struct {
 	// peerRequestQueue is a priority queue of requests received from peers.
@@ -150,7 +186,7 @@ type Engine struct {
 	lock sync.RWMutex // protects the fields immediately below
 
 	// peerLedger saves which peers are waiting for a Cid
-	peerLedger *peerLedger
+	peerLedger PeerLedger
 
 	// an external ledger dealing with peer scores
 	scoreLedger ScoreLedger
@@ -240,6 +276,13 @@ func WithScoreLedger(scoreledger ScoreLedger) Option {
 	}
 }
 
+// WithPeerLedger sets a custom [PeerLedger] to be used with this [Engine].
+func WithPeerLedger(peerLedger PeerLedger) Option {
+	return func(e *Engine) {
+		e.peerLedger = peerLedger
+	}
+}
+
 // WithBlockstoreWorkerCount sets the number of worker threads used for
 // blockstore operations in the decision engine
 func WithBlockstoreWorkerCount(count int) Option {
@@ -359,7 +402,7 @@ func newEngine(
 		taskWorkerCount:                 defaults.BitswapEngineTaskWorkerCount,
 		sendDontHaves:                   true,
 		self:                            self,
-		peerLedger:                      newPeerLedger(),
+		peerLedger:                      NewDefaultPeerLedger(),
 		pendingGauge:                    bmetrics.PendingEngineGauge(ctx),
 		activeGauge:                     bmetrics.ActiveEngineGauge(ctx),
 		targetMessageSize:               defaultTargetMessageSize,