Use v1 CIDs by default

README.md

@@ -10,13 +10,16 @@
 
 ```sh
 # calculate the CID of the Bored Ape 0 image
-curl -X POST --data-binary @ape0.png http://localhost:8001/api/cid/calculate
+curl -X POST -H 'Content-Type: application/octet-stream' --data-binary @ape0.png http://localhost:8001/api/cid/calculate
 # Upload Bored Ape 0 image
-curl -X POST --data-binary @ape0.png http://localhost:8001/api/upload/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ
+curl -X POST -H 'Content-Type: application/octet-stream' --data-binary @ape0.png http://localhost:8001/api/upload/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ
 "QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ"
 curl http://localhost:8080/ipfs/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ
 ```
 <image width="100px" src="https://ipfs.io/ipfs/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ" />
 
 ## todo
 - [ ] IPLD / directory support, eg: ipfs://CID/images/cat.png, ipfs://QmeSjSinHpPnmXmspMjwiXyN6zS4E9zccariGR3jxcaWtq/0
+- [ ] IPNS support
+- [ ] Better PIN support
+- [ ] CAR support

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/libp2p/go-libp2p-kad-dht v0.24.4
 	github.com/multiformats/go-multiaddr v0.11.0
 	github.com/multiformats/go-multibase v0.2.0
+	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
 	go.sia.tech/jape v0.11.0
 	go.sia.tech/renterd v0.6.1-0.20231117113258-d4afcc97a585
@@ -117,7 +118,6 @@ require (
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-multicodec v0.9.0 // indirect
 	github.com/multiformats/go-multistream v0.4.1 // indirect
 	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect

http/api.go

@@ -1,9 +1,12 @@
 package http
 
 import (
+	"errors"
 	"net/http"
 
 	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multicodec"
+	"github.com/multiformats/go-multihash"
 	"go.sia.tech/fsd/config"
 	"go.sia.tech/fsd/ipfs"
 	"go.sia.tech/fsd/sia"
@@ -29,7 +32,20 @@ func (as *apiServer) handleCalculate(jc jape.Context) {
 	body := jc.Request.Body
 	defer body.Close()
 
-	blocks, err := as.sia.CalculateBlocks(ctx, body)
+	opts := sia.CIDOptions{
+		CIDBuilder: cid.V1Builder{Codec: uint64(multicodec.DagPb), MhType: multihash.SHA2_256},
+		RawLeaves:  true,
+	}
+
+	if err := jc.DecodeForm("rawLeaves", &opts.RawLeaves); err != nil {
+		return
+	} else if err := jc.DecodeForm("maxLinks", &opts.MaxLinks); err != nil {
+		return
+	} else if err := jc.DecodeForm("blockSize", &opts.BlockSize); err != nil {
+		return
+	}
+
+	blocks, err := as.sia.CalculateBlocks(ctx, body, opts)
 	if err != nil {
 		jc.Error(err, http.StatusInternalServerError)
 		return
@@ -43,20 +59,32 @@ func (as *apiServer) handlePin(jc jape.Context) {
 	if err := jc.DecodeParam("cid", &cidStr); err != nil {
 		return
 	}
-	cid, err := cid.Parse(cidStr)
+	c, err := cid.Parse(cidStr)
 	if err != nil {
 		jc.Error(err, http.StatusBadRequest)
 		return
 	}
 
-	r, err := as.ipfs.DownloadCID(ctx, cid, nil)
+	// TODO: break this out for better support, the current implementation will
+	// not handle anything but standard unixfs files with the default block size
+	r, err := as.ipfs.DownloadCID(ctx, c, nil)
 	if err != nil {
 		jc.Error(err, http.StatusInternalServerError)
 		return
 	}
 	defer r.Close()
 
-	if err := as.sia.UploadCID(ctx, cid, r); err != nil {
+	var opts sia.CIDOptions
+	switch c.Version() {
+	case 1:
+		prefix := c.Prefix()
+		opts.CIDBuilder = cid.V1Builder{Codec: prefix.Codec, MhType: prefix.MhType, MhLength: prefix.MhLength}
+		opts.RawLeaves = true
+	case 0:
+		opts.CIDBuilder = cid.V0Builder{}
+	}
+
+	if err := as.sia.UploadCID(ctx, c, r, opts); err != nil {
 		jc.Error(err, http.StatusInternalServerError)
 		return
 	}
@@ -68,23 +96,40 @@ func (as *apiServer) handleUpload(jc jape.Context) {
 	if err := jc.DecodeParam("cid", &cidStr); err != nil {
 		return
 	}
-	cid, err := cid.Parse(cidStr)
+	c, err := cid.Parse(cidStr)
 	if err != nil {
 		jc.Error(err, http.StatusBadRequest)
 		return
+	} else if c.Version() != 1 {
+		jc.Error(errors.New("only v1 CIDs are supported"), http.StatusBadRequest)
+		return
 	}
 
 	body := jc.Request.Body
 	defer body.Close()
 
-	err = as.sia.UploadCID(ctx, cid, body)
+	prefix := c.Prefix()
+	opts := sia.CIDOptions{
+		CIDBuilder: cid.V1Builder{Codec: prefix.Codec, MhType: prefix.MhType, MhLength: prefix.MhLength},
+		RawLeaves:  true,
+	}
+
+	if err := jc.DecodeForm("rawLeaves", &opts.RawLeaves); err != nil {
+		return
+	} else if err := jc.DecodeForm("maxLinks", &opts.MaxLinks); err != nil {
+		return
+	} else if err := jc.DecodeForm("blockSize", &opts.BlockSize); err != nil {
+		return
+	}
+
+	err = as.sia.UploadCID(ctx, c, body, opts)
 	if err != nil {
 		jc.Error(err, http.StatusInternalServerError)
 		return
 	}
 
 	// the root cid is the first block
-	jc.Encode(cid.String())
+	jc.Encode(c.String())
 }
 
 func (as *apiServer) handleVerifyCID(jc jape.Context) {

sia/sia.go

@@ -44,13 +44,31 @@ type (
 
 		renterd config.Renterd
 	}
+
+	// CIDOptions holds configuration options for CID uploads
+	CIDOptions struct {
+		CIDBuilder cid.Builder
+		RawLeaves  bool
+		MaxLinks   int
+		BlockSize  int64
+	}
 )
 
 // ErrNotFound is returned when a CID is not found in the store
 var ErrNotFound = errors.New("not found")
 
+func setDefaultCIDOpts(opts *CIDOptions) {
+	if opts.MaxLinks <= 0 {
+		opts.MaxLinks = ihelpers.DefaultLinksPerBlock
+	}
+
+	if opts.BlockSize <= 0 {
+		opts.BlockSize = chunker.DefaultBlockSize
+	}
+}
+
 // UploadCID uploads a CID to the renterd node
-func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error {
+func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader, opts CIDOptions) error {
 	log := n.log.Named("upload").With(zap.Stringer("cid", c), zap.String("bucket", n.renterd.Bucket))
 
 	dataKey := c.String()
@@ -86,11 +104,13 @@ func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error {
 		}
 	}()
 
-	spl := chunker.NewSizeSplitter(r, chunker.DefaultBlockSize)
-
+	setDefaultCIDOpts(&opts)
+	spl := chunker.NewSizeSplitter(r, opts.BlockSize)
 	dbp := ihelpers.DagBuilderParams{
-		Maxlinks: ihelpers.DefaultLinksPerBlock,
-		Dagserv:  dagSvc,
+		Dagserv:    dagSvc,
+		CidBuilder: opts.CIDBuilder,
+		RawLeaves:  opts.RawLeaves,
+		Maxlinks:   opts.MaxLinks,
 	}
 	db, err := dbp.New(spl)
 	if err != nil {
@@ -189,14 +209,16 @@ func (n *Node) ProxyHTTPDownload(cid cid.Cid, r *http.Request, w http.ResponseWr
 }
 
 // CalculateBlocks calculates the blocks for a given reader and returns them
-func (n *Node) CalculateBlocks(ctx context.Context, r io.Reader) ([]Block, error) {
+func (n *Node) CalculateBlocks(ctx context.Context, r io.Reader, opts CIDOptions) ([]Block, error) {
 	dagSvc := NewUnixFileUploader("", io.Discard, io.Discard, n.log.Named("calculate"))
 
-	spl := chunker.NewSizeSplitter(r, chunker.DefaultBlockSize)
-
+	setDefaultCIDOpts(&opts)
+	spl := chunker.NewSizeSplitter(r, opts.BlockSize)
 	dbp := ihelpers.DagBuilderParams{
-		Maxlinks: ihelpers.DefaultLinksPerBlock,
-		Dagserv:  dagSvc,
+		Dagserv:    dagSvc,
+		CidBuilder: opts.CIDBuilder,
+		RawLeaves:  opts.RawLeaves,
+		Maxlinks:   opts.MaxLinks,
 	}
 	db, err := dbp.New(spl)
 	if err != nil {