CBG-3909: use deltas for pv and mv when persisting to the bucket (#7096)

* CBG-3909: use dletas for pv and mv when peristing to the bucket * tidy up * fix lint * remove comment * changes to reflect xdcr format * updates based off review * clean up * fix from rebase * safety for decoding delta value * lint error Signed-off-by: Gregory Newman-Smith <[email protected]> * updates to remove leading 0x in deltas * fix comment * custom marahal/unmarahal * lint fix * lint * updates to marshal and unmarshal functions * remove unused param * updated to add new test cases and small changes * small update --------- Signed-off-by: Gregory Newman-Smith <[email protected]>
couchbase · Oct 11, 2024 · 7ad9dbf · 7ad9dbf
1 parent d7d7a7a
commit 7ad9dbf
Show file tree

Hide file tree

Showing 8 changed files with 451 additions and 97 deletions.
diff --git a/base/util.go b/base/util.go
@@ -1018,6 +1018,49 @@ func HexCasToUint64(cas string) uint64 {
 	return binary.LittleEndian.Uint64(casBytes[0:8])
 }
 
+// HexCasToUint64ForDelta will convert hex cas to uint64 accounting for any stripped zeros in delta calculation
+func HexCasToUint64ForDelta(casByte []byte) (uint64, error) {
+	var decoded []byte
+
+	// as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length
+	if len(casByte)%2 != 0 {
+		casByte = append(casByte, '0')
+	}
+
+	// create byte array for decoding into
+	decodedLen := hex.DecodedLen(len(casByte))
+	// binary.LittleEndian.Uint64 expects length 8 byte array, if larger we should error, if smaller
+	// (because of stripped 0's then we should make it length 8).
+	if decodedLen > 8 {
+		return 0, fmt.Errorf("corrupt hex value, decoded length larger than expected")
+	}
+	if decodedLen < 8 {
+		// can be less than 8 given we have stripped the 0's for some values, in this case we need to ensure large eniough
+		decoded = make([]byte, 8)
+	} else {
+		decoded = make([]byte, decodedLen)
+	}
+
+	if _, err := hex.Decode(decoded, casByte); err != nil {
+		return 0, err
+	}
+	res := binary.LittleEndian.Uint64(decoded)
+	return res, nil
+}
+
+// Uint64ToLittleEndianHexAndStripZeros will convert a uint64 type to little endian hex, stripping any zeros off the end
+// + stripping 0x from start
+func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string {
+	hexCas := Uint64CASToLittleEndianHex(cas)
+
+	i := len(hexCas) - 1
+	for i > 2 && hexCas[i] == '0' {
+		i--
+	}
+	// strip 0x from start
+	return string(hexCas[2 : i+1])
+}
+
 func HexToBase64(s string) ([]byte, error) {
 	decoded := make([]byte, hex.DecodedLen(len(s)))
 	if _, err := hex.Decode(decoded, []byte(s)); err != nil {

diff --git a/base/util_test.go b/base/util_test.go
@@ -1735,3 +1735,40 @@ func TestCASToLittleEndianHex(t *testing.T) {
 	littleEndianHex := Uint64CASToLittleEndianHex(casValue)
 	require.Equal(t, expHexValue, string(littleEndianHex))
 }
+
+func TestUint64CASToLittleEndianHexAndStripZeros(t *testing.T) {
+	hexLE := "0x0000000000000000"
+	u64 := HexCasToUint64(hexLE)
+	hexLEStripped := Uint64ToLittleEndianHexAndStripZeros(u64)
+	u64Stripped, err := HexCasToUint64ForDelta([]byte(hexLEStripped))
+	require.NoError(t, err)
+	assert.Equal(t, u64, u64Stripped)
+
+	hexLE = "0xffffffffffffffff"
+	u64 = HexCasToUint64(hexLE)
+	hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64)
+	u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped))
+	require.NoError(t, err)
+	assert.Equal(t, u64, u64Stripped)
+
+	hexLE = "0xd123456e789a0bcf"
+	u64 = HexCasToUint64(hexLE)
+	hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64)
+	u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped))
+	require.NoError(t, err)
+	assert.Equal(t, u64, u64Stripped)
+
+	hexLE = "0xd123456e78000000"
+	u64 = HexCasToUint64(hexLE)
+	hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64)
+	u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped))
+	require.NoError(t, err)
+	assert.Equal(t, u64, u64Stripped)
+
+	hexLE = "0xa500000000000000"
+	u64 = HexCasToUint64(hexLE)
+	hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64)
+	u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped))
+	require.NoError(t, err)
+	assert.Equal(t, u64, u64Stripped)
+}
diff --git a/db/document.go b/db/document.go
@@ -1117,7 +1117,8 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat
 			}
 		}
 		if hlvXattrData != nil {
-			err := base.JSONUnmarshal(hlvXattrData, &doc.SyncData.HLV)
+			// parse the raw bytes of the hlv and convert deltas back to full values in memory
+			err := base.JSONUnmarshal(hlvXattrData, &doc.HLV)
 			if err != nil {
 				return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync).  Error: %v", base.UD(doc.ID), err))
 			}
@@ -1157,7 +1158,8 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat
 			}
 		}
 		if hlvXattrData != nil {
-			err := base.JSONUnmarshal(hlvXattrData, &doc.SyncData.HLV)
+			// parse the raw bytes of the hlv and convert deltas back to full values in memory
+			err := base.JSONUnmarshal(hlvXattrData, &doc.HLV)
 			if err != nil {
 				return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory).  Error: %v", base.UD(doc.ID), err))
 			}
@@ -1251,7 +1253,7 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl
 		}
 	}
 	if doc.SyncData.HLV != nil {
-		vvXattr, err = base.JSONMarshal(&doc.SyncData.HLV)
+		vvXattr, err = base.JSONMarshal(doc.SyncData.HLV)
 		if err != nil {
 			return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s.  Error: %v", base.UD(doc.ID), err))
 		}

diff --git a/db/document_test.go b/db/document_test.go
@@ -239,24 +239,16 @@ const doc_meta_no_vv = `{
     "time_saved": "2017-10-25T12:45:29.622450174-07:00"
   }`
 
-const doc_meta_vv = `{
-	"cvCas":"0x40e2010000000000",
-	"src":"cb06dc003846116d9b66d2ab23887a96",
-	"ver":"0x40e2010000000000",
-	"mv":{
-		"s_LhRPsa7CpjEvP5zeXTXEBA":"c0ff05d7ac059a16",
-		"s_NqiIe0LekFPLeX4JvTO6Iw":"1c008cd6ac059a16"
-	},
-	"pv":{
-		"s_YZvBpEaztom9z5V/hDoeIw":"f0ff44d6ac059a16"
-	}
-  }`
+const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000",
+	"mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"],
+	"pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"]
+}`
 
 func TestParseVersionVectorSyncData(t *testing.T) {
 	mv := make(HLVVersions)
 	pv := make(HLVVersions)
-	mv["s_LhRPsa7CpjEvP5zeXTXEBA"] = 1628620455147864000 //"c0ff05d7ac059a16"
-	mv["s_NqiIe0LekFPLeX4JvTO6Iw"] = 1628620455139868700
+	mv["s_LhRPsa7CpjEvP5zeXTXEBA"] = 1628620455147864000
+	mv["s_NqiIe0LekFPLeX4JvTO6Iw"] = 1628620458747363292
 	pv["s_YZvBpEaztom9z5V/hDoeIw"] = 1628620455135215600
 
 	ctx := base.TestCtx(t)
@@ -295,6 +287,22 @@ func TestParseVersionVectorSyncData(t *testing.T) {
 	assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions))
 }
 
+const doc_meta_vv_corrupt = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000",
+	"mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd61c008cd61c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"],
+	"pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"]
+}`
+
+func TestParseVersionVectorCorruptDelta(t *testing.T) {
+
+	ctx := base.TestCtx(t)
+
+	sync_meta := []byte(doc_meta_no_vv)
+	vv_meta := []byte(doc_meta_vv_corrupt)
+	_, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll)
+	require.Error(t, err)
+
+}
+
 // TestRevAndVersion tests marshalling and unmarshalling rev and current version
 func TestRevAndVersion(t *testing.T) {