Merge pull request #669 from ipfs/release-v0.23.0

Release v0.23.0

.github/workflows/gateway-sharness.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Replace boxo in Kubo go.mod
         run: |
           go mod edit -replace=github.com/ipfs/boxo=../boxo
-          go mod tidy
+          make mod_tidy
           cat go.mod
         working-directory: kubo
       - name: Install sharness dependencies

CHANGELOG.md

@@ -24,26 +24,38 @@ The following emojis are used to highlight certain changes:
 
 ### Security
 
-## [v0.22.0]
+## [v0.23.0]
+
+### Added
+
+- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata [#653](https://github.com/ipfs/boxo/pull/653)
+- `gateway` deserialized responses will have `Last-Modified` set to value from optional UnixFS 1.5 modification time field (if present in DAG) and a matching `If-Modified-Since` will return `304 Not Modified` (UnixFS 1.5 files only) [#659](https://github.com/ipfs/boxo/pull/659)
 
 ### Changed
 
-- `go-libp2p` dependency updated to [v0.36 (release notes)](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.1)
-- `bitswap/server` minor memory use and performance improvements
-- `bitswap` unify logger names to use uniform format bitswap/path/pkgname
-- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories
-- generate random test data using `ipfs/go-test` instead of internal util code
+- updated Go in `go.mod` to 1.22 [#661](https://github.com/ipfs/boxo/pull/661)
+- updated go-libp2p to [v0.36.3](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.3)
+- `chunker` refactored to reduce overall memory use by reducing heap fragmentation [#649](https://github.com/ipfs/boxo/pull/649)
+- `bitswap/server` minor performance improvements in concurrent operations [#666](https://github.com/ipfs/boxo/pull/666)
+- removed dependency on go-ipfs-blocksutil [#656](https://github.com/ipfs/boxo/pull/656)
 
-### Removed
+## [v0.22.0]
 
-- `util` logic for generating random test data moved to [`ipfs/go-test/random`](https://github.com/ipfs/go-test)
+### Changed
+
+- `go-libp2p` dependency updated to [v0.36 (release notes)](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.1)
+- `bitswap/server` minor memory use and performance improvements [#634](https://github.com/ipfs/boxo/pull/634)
+- `bitswap` unify logger names to use uniform format bitswap/path/pkgname [#637](https://github.com/ipfs/boxo/pull/637)
+- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories [#643](https://github.com/ipfs/boxo/pull/643)
+- `util` generate random test data using `ipfs/go-test` instead of internal util code [#638](https://github.com/ipfs/boxo/pull/638)
+- `bitswap/server` `PeerLedger.Wants` now returns `bool` (interface change from `Wants(p peer.ID, e wl.Entry)` to `Wants(p peer.ID, e wl.Entry) bool`) [#629](https://github.com/ipfs/boxo/pull/629)
 
 ### Fixed
 
-- `boxo/gateway` now correctly returns 404 Status Not Found instead of 500 when the requested content cannot be found due to offline exchange, gateway running in no-fetch (non-recursive) mode, or a similar restriction that only serves a specific set of CIDs.
-- `bitswap/client` fix memory leak in BlockPresenceManager due to unlimited map growth.
-- `bitswap/network` fixed race condition when a timeout occurred before hole punching completed while establishing a first-time stream to a peer behind a NAT
-- `bitswap`: wantlist overflow handling now cancels existing entries to make room for newer entries. This fix prevents the wantlist from filling up with CIDs that the server does not have.
+- `boxo/gateway` now correctly returns 404 Status Not Found instead of 500 when the requested content cannot be found due to offline exchange, gateway running in no-fetch (non-recursive) mode, or a similar restriction that only serves a specific set of CIDs. [#630](https://github.com/ipfs/boxo/pull/630)
+- `bitswap/client` fix memory leak in BlockPresenceManager due to unlimited map growth. [#636](https://github.com/ipfs/boxo/pull/636)
+- `bitswap/network` fixed race condition when a timeout occurred before hole punching completed while establishing a first-time stream to a peer behind a NAT [#651](https://github.com/ipfs/boxo/pull/651)
+- `bitswap`: wantlist overflow handling now cancels existing entries to make room for newer entries. This fix prevents the wantlist from filling up with CIDs that the server does not have. [#629](https://github.com/ipfs/boxo/pull/629)
 
 ## [v0.21.0]
 

README.md

@@ -156,22 +156,23 @@ The code coverage of this repo is not currently representative of the actual tes
 
 ### Help
 
-If you have questions, feel free to open an issue. You can also find the Boxo maintainers in [Filecoin Slack](https://filecoin.io/slack/) at #Boxo-maintainers.  (If you would like to engage via IPFS Discord or ipfs.io Matrix, please drop into the #ipfs-implementers channel/room or file an issue, and we'll get bridging from #Boxo-maintainers to these other chat platforms.)
+If you suspect a bug or have technical questions, feel free to open an issue. 
+
+For regular support, try [Community chat](https://docs.ipfs.tech/community/#chat)'s `#ipfs-implementers` room or [help/boxo at IPFS discussion forums](https://discuss.ipfs.tech/c/help/boxo/51).
 
 ### What is the response time for issues or PRs filed?
 
-New issues and PRs to this repo are usually looked at on a weekly basis as part of [Kubo triage](https://pl-strflt.notion.site/Kubo-Issue-Triage-Notes-7d4983e8cf294e07b3cc51b0c60ede9a). However, the response time may vary.
+New issues and PRs to this repo are usually looked at on a weekly basis as part of [Shipyard's GO Triage triage](https://ipshipyard.notion.site/IPFS-Go-Triage-Boxo-Kubo-Rainbow-0ddee6b7f28d412da7dabe4f9107c29a). However, the response time may vary.
 
 ### What are some projects that depend on this project?
 
 The exhaustive list is https://github.com/ipfs/boxo/network/dependents. Some notable projects include:
 
 1. [Kubo](https://github.com/ipfs/kubo), an IPFS implementation in Go
 2. [Lotus](https://github.com/filecoin-project/lotus), a Filecoin implementation in Go
-6. [rainbow](https://github.com/ipfs/rainbow), a specialized IPFS gateway
-4. [ipfs-check](https://github.com/ipfs-shipyard/ipfs-check), checks IPFS data availability
-5. [someguy](https://github.com/ipfs-shipyard/someguy), a dedicated Delegated Routing V1 server and client
-3. [Bifrost Gateway](https://github.com/ipfs/bifrost-gateway), a dedicated IPFS Gateway daemon backed by a remote datastore
+3. [rainbow](https://github.com/ipfs/rainbow), a specialized IPFS gateway
+4. [ipfs-check](https://github.com/ipfs/ipfs-check), checks IPFS data availability
+5. [someguy](https://github.com/ipfs/someguy), a dedicated Delegated Routing V1 server and client
 
 ### Governance and Access
 

RELEASE.md

@@ -52,7 +52,7 @@ At least as of 2023-06-08, changelog test is manually copied from [the changelog
 ### Related Work
 Below are links of related/adjacent work that has informed some of the decisions in this document:
 1. https://github.com/ipfs/boxo/issues/170
-2. https://pl-strflt.notion.site/Kubo-Release-Process-5a5d066264704009a28a79cff93062c4
+2. https://ipshipyard.notion.site/Kubo-Release-Process-6dba4f5755c9458ab5685eeb28173778
 3. https://github.com/ipfs/kubo/blob/master/docs/RELEASE_ISSUE_TEMPLATE.md
 
 ## Release Process
@@ -73,7 +73,7 @@ Below are links of related/adjacent work that has informed some of the decisions
 - [ ] Ensure Boxo tests are passing
 - [ ] Ensure Kubo tests are passing
   - [ ] Go to Kubo dir and run `go get github.com/ipfs/boxo@<commit_hash>` using the commit hash of the `release-vX.Y.Z` branch
-  - [ ] Run `go mod tidy` in repo root and in `docs/examples/kubo-as-a-library`
+  - [ ] Run `make mod_tidy` in repo root (to apply `go mod tidy` to code, tests, and examples)
   - [ ] Commit the changes and open a draft PR in Kubo
   - [ ] Name the PR "Upgrade to Boxo vX.Y.Z"
   - [ ] Paste a link to the Kubo PR in the Boxo PR, so reviewers can verify the Kubo test run

bitswap/bitswap_test.go

@@ -19,14 +19,16 @@ import (
 	blocks "github.com/ipfs/go-block-format"
 	cid "github.com/ipfs/go-cid"
 	detectrace "github.com/ipfs/go-detect-race"
-	blocksutil "github.com/ipfs/go-ipfs-blocksutil"
 	delay "github.com/ipfs/go-ipfs-delay"
 	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-test/random"
 	tu "github.com/libp2p/go-libp2p-testing/etc"
 	p2ptestutil "github.com/libp2p/go-libp2p-testing/netutil"
 	peer "github.com/libp2p/go-libp2p/core/peer"
 )
 
+const blockSize = 4
+
 func isCI() bool {
 	// https://github.blog/changelog/2020-04-15-github-actions-sets-the-ci-environment-variable-to-true/
 	return os.Getenv("CI") != ""
@@ -52,9 +54,7 @@ func TestClose(t *testing.T) {
 	vnet := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(vnet, nil, nil)
 	defer ig.Close()
-	bgen := blocksutil.NewBlockGenerator()
-
-	block := bgen.Next()
+	block := random.BlocksOfSize(1, blockSize)[0]
 	bitswap := ig.Next()
 
 	bitswap.Exchange.Close()
@@ -187,7 +187,6 @@ func TestUnwantedBlockNotAdded(t *testing.T) {
 func TestPendingBlockAdded(t *testing.T) {
 	ctx := context.Background()
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
-	bg := blocksutil.NewBlockGenerator()
 	sessionBroadcastWantCapacity := 4
 
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
@@ -202,7 +201,7 @@ func TestPendingBlockAdded(t *testing.T) {
 	// Request enough blocks to exceed the session's broadcast want list
 	// capacity (by one block). The session will put the remaining block
 	// into the "tofetch" queue
-	blks := bg.Blocks(sessionBroadcastWantCapacity + 1)
+	blks := random.BlocksOfSize(sessionBroadcastWantCapacity+1, blockSize)
 	ks := make([]cid.Cid, 0, len(blks))
 	for _, b := range blks {
 		ks = append(ks, b.Cid())
@@ -285,10 +284,9 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
 		bitswap.MaxOutstandingBytesPerPeer(1 << 20),
 	})
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	instances := ig.Instances(numInstances)
-	blocks := bg.Blocks(numBlocks)
+	blocks := random.BlocksOfSize(numBlocks, blockSize)
 
 	t.Log("Give the blocks to the first instance")
 
@@ -338,7 +336,6 @@ func TestSendToWantingPeer(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	peers := ig.Instances(2)
 	peerA := peers[0]
@@ -349,7 +346,7 @@ func TestSendToWantingPeer(t *testing.T) {
 
 	waitTime := time.Second * 5
 
-	alpha := bg.Next()
+	alpha := random.BlocksOfSize(1, blockSize)[0]
 	// peerA requests and waits for block alpha
 	ctx, cancel := context.WithTimeout(context.Background(), waitTime)
 	defer cancel()
@@ -409,12 +406,11 @@ func TestBasicBitswap(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	t.Log("Test a one node trying to get one block from another")
 
 	instances := ig.Instances(3)
-	blocks := bg.Blocks(1)
+	blocks := random.BlocksOfSize(1, blockSize)
 
 	// First peer has block
 	addBlock(t, context.Background(), instances[0], blocks[0])
@@ -481,12 +477,11 @@ func TestDoubleGet(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	t.Log("Test a one node trying to get one block from another")
 
 	instances := ig.Instances(2)
-	blocks := bg.Blocks(1)
+	blocks := random.BlocksOfSize(1, blockSize)
 
 	// NOTE: A race condition can happen here where these GetBlocks requests go
 	// through before the peers even get connected. This is okay, bitswap
@@ -546,12 +541,11 @@ func TestWantlistCleanup(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	instances := ig.Instances(2)
 	instance := instances[0]
 	bswap := instance.Exchange
-	blocks := bg.Blocks(20)
+	blocks := random.BlocksOfSize(20, blockSize)
 
 	var keys []cid.Cid
 	for _, b := range blocks {
@@ -668,12 +662,11 @@ func TestBitswapLedgerOneWay(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	t.Log("Test ledgers match when one peer sends block to another")
 
 	instances := ig.Instances(2)
-	blocks := bg.Blocks(1)
+	blocks := random.BlocksOfSize(1, blockSize)
 	addBlock(t, context.Background(), instances[0], blocks[0])
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
@@ -693,12 +686,12 @@ func TestBitswapLedgerOneWay(t *testing.T) {
 	}
 
 	// check that receipts have intended values
-	ratest := newReceipt(1, 0, 1)
+	ratest := newReceipt(blockSize, 0, 1)
 	err = assertLedgerEqual(ratest, ra)
 	if err != nil {
 		t.Fatal(err)
 	}
-	rbtest := newReceipt(0, 1, 1)
+	rbtest := newReceipt(0, blockSize, 1)
 	err = assertLedgerEqual(rbtest, rb)
 	if err != nil {
 		t.Fatal(err)
@@ -717,12 +710,11 @@ func TestBitswapLedgerTwoWay(t *testing.T) {
 	net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
 	ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 	defer ig.Close()
-	bg := blocksutil.NewBlockGenerator()
 
 	t.Log("Test ledgers match when two peers send one block to each other")
 
 	instances := ig.Instances(2)
-	blocks := bg.Blocks(2)
+	blocks := random.BlocksOfSize(2, blockSize)
 	addBlock(t, context.Background(), instances[0], blocks[0])
 	addBlock(t, context.Background(), instances[1], blocks[1])
 
@@ -750,7 +742,7 @@ func TestBitswapLedgerTwoWay(t *testing.T) {
 	}
 
 	// check that receipts have intended values
-	rtest := newReceipt(1, 1, 2)
+	rtest := newReceipt(blockSize, blockSize, 2)
 	err = assertLedgerEqual(rtest, ra)
 	if err != nil {
 		t.Fatal(err)