Tusd PATCH Checksums

.drone.env

@@ -1,4 +1,4 @@
 # The test runner source for API tests
-APITESTS_COMMITID=37491c15674ab02347cbf6c90124002e75afa339
+APITESTS_COMMITID=5120b86d92f69c04f999697f7b584379327d62cb
 APITESTS_BRANCH=master
 APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git

changelog/unreleased/tusd-checksums.md

@@ -0,0 +1,5 @@
+Enhancement: Tusd PATCH checksums
+
+Check checksums also on chunked uploads during PATCH requests
+
+https://github.com/cs3org/reva/pull/4807

internal/http/services/owncloud/ocdav/options.go

@@ -42,7 +42,7 @@ func (s *svc) handleOptions(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set(net.HeaderTusResumable, "1.0.0") // TODO(jfd): only for dirs?
 		w.Header().Set(net.HeaderTusVersion, "1.0.0")
 		w.Header().Set(net.HeaderTusExtension, "creation,creation-with-upload,checksum,expiration")
-		w.Header().Set(net.HeaderTusChecksumAlgorithm, "md5,sha1,crc32")
+		w.Header().Set(net.HeaderTusChecksumAlgorithm, "md5,sha1,adler32")
 	}
 	w.WriteHeader(http.StatusNoContent)
 }

pkg/ctx/checksumctx.go

@@ -0,0 +1,23 @@
+package ctx
+
+import "context"
+
+// ContextGetChecksum returns the checksum if set in the given context.
+func ContextGetChecksum(ctx context.Context) (string, bool) {
+	u, ok := ctx.Value(checksumKey).(string)
+	return u, ok
+}
+
+// ContextWithChecksum returns the checksum if set in the given context. Otherwise it panics.
+func ContextMustGetChecksum(ctx context.Context) string {
+	u, ok := ctx.Value(checksumKey).(string)
+	if !ok {
+		panic("checksum not set in context")
+	}
+	return u
+}
+
+// ContextSetChecksum returns a new context with the given checksum.
+func ContextSetChecksum(ctx context.Context, checksum string) context.Context {
+	return context.WithValue(ctx, checksumKey, checksum)
+}

pkg/ctx/userctx.go

@@ -33,6 +33,7 @@ const (
 	lockIDKey
 	scopeKey
 	initiatorKey
+	checksumKey
 )
 
 // ContextGetUser returns the user if set in the given context.

pkg/rhttp/datatx/manager/tus/tus.go

@@ -31,6 +31,7 @@ import (
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net"
 	"github.com/cs3org/reva/v2/pkg/appctx"
+	ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
 	"github.com/cs3org/reva/v2/pkg/errtypes"
 	"github.com/cs3org/reva/v2/pkg/events"
 	"github.com/cs3org/reva/v2/pkg/rhttp/datatx"
@@ -174,6 +175,10 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
 			}()
 			// set etag, mtime and file id
 			setHeaders(fs, w, r)
+			// set checksum
+			if v := r.Header.Get("Upload-Checksum"); v != "" {
+				r = r.WithContext(ctxpkg.ContextSetChecksum(r.Context(), v))
+			}
 			handler.PatchFile(w, r)
 		case "DELETE":
 			handler.DelFile(w, r)

pkg/storage/utils/decomposedfs/node/node.go

@@ -1354,10 +1354,6 @@ func enoughDiskSpace(path string, fileSize uint64) bool {
 
 // CalculateChecksums calculates the sha1, md5 and adler32 checksums of a file
 func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) {
-	sha1h := sha1.New()
-	md5h := md5.New()
-	adler32h := adler32.New()
-
 	_, subspan := tracer.Start(ctx, "os.Open")
 	f, err := os.Open(path)
 	subspan.End()
@@ -1366,11 +1362,20 @@ func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash,
 	}
 	defer f.Close()
 
-	r1 := io.TeeReader(f, sha1h)
+	return CalculateChecksumsFromReader(ctx, f)
+}
+
+// CalculateChecksumsFromReader calculates the sha1, md5 and adler32 checksums of a io.Reader
+func CalculateChecksumsFromReader(ctx context.Context, r io.Reader) (hash.Hash, hash.Hash, hash.Hash32, error) {
+	sha1h := sha1.New()
+	md5h := md5.New()
+	adler32h := adler32.New()
+
+	r1 := io.TeeReader(r, sha1h)
 	r2 := io.TeeReader(r1, md5h)
 
-	_, subspan = tracer.Start(ctx, "io.Copy")
-	_, err = io.Copy(adler32h, r2)
+	_, subspan := tracer.Start(ctx, "io.Copy")
+	_, err := io.Copy(adler32h, r2)
 	subspan.End()
 	if err != nil {
 		return nil, nil, nil, err

pkg/storage/utils/decomposedfs/upload/upload.go

@@ -19,7 +19,9 @@
 package upload
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -57,6 +59,22 @@ var defaultFilePerm = os.FileMode(0664)
 func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
 	ctx, span := tracer.Start(session.Context(ctx), "WriteChunk")
 	defer span.End()
+
+	// calculate checksum here
+	if checksum, ok := ctxpkg.ContextGetChecksum(ctx); ok {
+		// we need to copy the contents into memory so we can write it to disk later
+		b := bytes.NewBuffer(nil)
+		sha1, md5, adler32, err := node.CalculateChecksumsFromReader(ctx, io.TeeReader(src, b))
+		if err != nil {
+			return 0, err
+		}
+
+		if err := verifyChecksum(checksum, sha1, md5, adler32, true); err != nil {
+			return 0, err
+		}
+		src = b
+	}
+
 	_, subspan := tracer.Start(ctx, "os.OpenFile")
 	file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
 	subspan.End()
@@ -65,10 +83,6 @@ func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io
 	}
 	defer file.Close()
 
-	// calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum
-	// TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ...
-	// It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used
-	// but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ...
 	_, subspan = tracer.Start(ctx, "io.Copy")
 	n, err := io.Copy(file, src)
 	subspan.End()
@@ -116,21 +130,7 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error {
 	// compare if they match the sent checksum
 	// TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads
 	if session.info.MetaData["checksum"] != "" {
-		var err error
-		parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2)
-		if len(parts) != 2 {
-			return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
-		}
-		switch parts[0] {
-		case "sha1":
-			err = checkHash(parts[1], sha1h)
-		case "md5":
-			err = checkHash(parts[1], md5h)
-		case "adler32":
-			err = checkHash(parts[1], adler32h)
-		default:
-			err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
-		}
+		err := verifyChecksum(session.info.MetaData["checksum"], sha1h, md5h, adler32h, false)
 		if err != nil {
 			session.store.Cleanup(ctx, session, true, false, false)
 			return err
@@ -264,10 +264,18 @@ func (session *OcisSession) Finalize() (err error) {
 	return nil
 }
 
-func checkHash(expected string, h hash.Hash) error {
+func checkHash(expected string, h hash.Hash, isBase64 bool) error {
 	hash := hex.EncodeToString(h.Sum(nil))
+	if isBase64 {
+		raw, err := base64.StdEncoding.DecodeString(expected)
+		if err != nil {
+			return err
+		}
+
+		expected = string(raw)
+	}
 	if expected != hash {
-		return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, hash))
+		return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %s", expected, hash))
 	}
 	return nil
 }
@@ -376,3 +384,20 @@ func joinurl(paths ...string) string {
 
 	return s.String()
 }
+
+func verifyChecksum(checksum string, sha1h hash.Hash, md5h hash.Hash, adler32h hash.Hash32, isBase64 bool) error {
+	parts := strings.SplitN(checksum, " ", 2)
+	if len(parts) != 2 {
+		return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
+	}
+	switch strings.ToLower(parts[0]) {
+	case "sha1":
+		return checkHash(parts[1], sha1h, isBase64)
+	case "md5":
+		return checkHash(parts[1], md5h, isBase64)
+	case "adler32":
+		return checkHash(parts[1], adler32h, isBase64)
+	default:
+		return errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
+	}
+}