From 4489b79719a6fa75b8770c45d611ead4ae79b07d Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 29 Aug 2024 15:18:59 +0100 Subject: [PATCH 01/19] CBG-3909: use dletas for pv and mv when peristing to the bucket --- base/constants.go | 10 ++++ base/util.go | 39 ++++++++++++++ base/util_test.go | 20 +++++++ db/document.go | 67 +++++++++++++++++++++-- db/document_test.go | 16 ++---- db/hybrid_logical_vector.go | 91 ++++++++++++++++++++++++++++++++ db/hybrid_logical_vector_test.go | 88 ++++++++++++++++++++++++++++++ db/utilities_hlv_testing.go | 21 ++++++++ go.sum | 15 ++++-- rest/api_test.go | 69 ++++++++++++++++++++++++ 10 files changed, 415 insertions(+), 21 deletions(-) diff --git a/base/constants.go b/base/constants.go index 49530a20ac..bb5aada41a 100644 --- a/base/constants.go +++ b/base/constants.go @@ -184,6 +184,16 @@ const ( SyncFnErrorMissingChannelAccess = "sg missing channel access" ) +const ( + HLV_CVCAS_FIELD = "cvCas" // This stores the version of the HLV + // These are the fields in the HLV + HLV_SRC_FIELD = "src" // source id for HLV + HLV_VER_FIELD = "ver" // ver field in HLV + HLV_MV_FIELD = "mv" // the MV field in HLV + HLV_PV_FIELD = "pv" // The PV field HLV + HLV_IMPORT_CAS = "importCAS" // import cas field +) + const ( // EmptyDocument denotes an empty document in JSON form. EmptyDocument = `{}` diff --git a/base/util.go b/base/util.go index a3dd50f27b..5afa8a3b66 100644 --- a/base/util.go +++ b/base/util.go @@ -48,6 +48,8 @@ const ( kMaxDeltaTtlDuration = 60 * 60 * 24 * 30 * time.Second ) +var MaxDecodedLength = len(Uint64CASToLittleEndianHex(math.MaxUint64)) + // NonCancellableContext is here to stroe a context that is not cancellable. Used to explicitly state when a change from // a cancellable context to a context withoutr contex is required type NonCancellableContext struct { @@ -1018,6 +1020,43 @@ func HexCasToUint64(cas string) uint64 { return binary.LittleEndian.Uint64(casBytes[0:8]) } +// HexCasToUint64ForDelta will convert hex cas to uint64 accounting for any stripped zeros in delta calculation +func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { + if len(casByte) <= 2 { + return 0, fmt.Errorf("hex value is too short.") + } + if casByte[0] != '0' || casByte[1] != 'x' { + return 0, fmt.Errorf("incorrect hex value, leading 0x is expected") + } + + // as we strip any zeros iff the end of the hex value for deltas, the input delta could be odd length + if len(casByte)%2 != 0 { + evenHexLen := make([]byte, len(casByte), len(casByte)+1) + copy(evenHexLen, casByte) + evenHexLen = append(evenHexLen, '0') + casByte = evenHexLen + } + + decoded := make([]byte, MaxDecodedLength) + _, err := hex.Decode(decoded, casByte[2:]) + if err != nil { + return 0, err + } + res := binary.LittleEndian.Uint64(decoded) + return res, nil +} + +// Uint64ToLittleEndianHexAndStripZeros will convert a uin64 type to little endian hex, stripping any zeros off the end +func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string { + hexCas := Uint64CASToLittleEndianHex(cas) + + i := len(hexCas) - 1 + for i > 2 && hexCas[i] == '0' { + i-- + } + return string(hexCas[:i+1]) +} + func HexToBase64(s string) ([]byte, error) { decoded := make([]byte, hex.DecodedLen(len(s))) if _, err := hex.Decode(decoded, []byte(s)); err != nil { diff --git a/base/util_test.go b/base/util_test.go index 7d8544682f..b37c67f647 100644 --- a/base/util_test.go +++ b/base/util_test.go @@ -1735,3 +1735,23 @@ func TestCASToLittleEndianHex(t *testing.T) { littleEndianHex := Uint64CASToLittleEndianHex(casValue) require.Equal(t, expHexValue, string(littleEndianHex)) } + +func TestUint64CASToLittleEndianHexAndStripZeros(t *testing.T) { + hexLE := "0x0000000000000000" + u64 := HexCasToUint64(hexLE) + hexLEStripped := Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped := HexCasToUint64(hexLEStripped) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xffffffffffffffff" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped = HexCasToUint64(hexLEStripped) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xd123456e789a0bcf" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped = HexCasToUint64(hexLEStripped) + assert.Equal(t, u64, u64Stripped) +} diff --git a/db/document.go b/db/document.go index 85ec9ebc4f..1f92999bd3 100644 --- a/db/document.go +++ b/db/document.go @@ -484,10 +484,11 @@ func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey syncXattr, ok := xattrValues[base.SyncXattrName] if vvXattr, ok := xattrValues[base.VvXattrName]; ok { - err = base.JSONUnmarshal(vvXattr, &hlv) + docHLV, err := ParseHLVFields(vvXattr) if err != nil { return nil, nil, nil, fmt.Errorf("error unmarshalling HLV: %w", err) } + hlv = docHLV } if ok && len(syncXattr) > 0 { @@ -1117,10 +1118,12 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } } if hlvXattrData != nil { - err := base.JSONUnmarshal(hlvXattrData, &doc.SyncData.HLV) + // parse the raw bytes of the hlv and convert deltas back to full values in memory + docHLV, err := ParseHLVFields(hlvXattrData) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) } + doc.HLV = docHLV } if virtualXattr != nil { var revSeqNo string @@ -1157,10 +1160,12 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } } if hlvXattrData != nil { - err := base.JSONUnmarshal(hlvXattrData, &doc.SyncData.HLV) + // parse the raw bytes of the hlv and convert deltas back to full values in memory + docHLV, err := ParseHLVFields(hlvXattrData) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), err)) } + doc.HLV = docHLV } if len(globalSyncData) > 0 { if err := base.JSONUnmarshal(globalSyncData, &doc.GlobalSyncData); err != nil { @@ -1251,7 +1256,8 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl } } if doc.SyncData.HLV != nil { - vvXattr, err = base.JSONMarshal(&doc.SyncData.HLV) + // convert in memory hlv into persisted version, converting maps to deltas + vvXattr, err = ConstructXattrFromHlv(doc.HLV) if err != nil { return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) } @@ -1286,6 +1292,59 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl return data, syncXattr, vvXattr, mouXattr, globalXattr, nil } +// ConstructXattrFromHlv will build a persisted hlv from teh in memory hlv. Converting the pv and mv maps to deltas +func ConstructXattrFromHlv(hlv *HybridLogicalVector) ([]byte, error) { + var persistedHLV PersistedHLV + + persistedHLV.SourceID = hlv.SourceID + persistedHLV.Version = hlv.Version + persistedHLV.ImportCAS = hlv.ImportCAS + persistedHLV.CurrentVersionCAS = hlv.CurrentVersionCAS + if len(hlv.PreviousVersions) > 0 { + persistedHLV.PreviousVersions = make([]string, len(hlv.PreviousVersions)) + persistedHLV.PreviousVersions = VersionsToDeltas(hlv.PreviousVersions) + } + if len(hlv.MergeVersions) > 0 { + persistedHLV.MergeVersions = make([]string, len(hlv.MergeVersions)) + persistedHLV.MergeVersions = VersionsToDeltas(hlv.MergeVersions) + } + + hlvData, err := base.JSONMarshal(&persistedHLV) + if err != nil { + return nil, err + } + return hlvData, nil +} + +// ParseHLVFields will parse raw bytes of the hlv xattr from persisted version to in memory version, converting the +// deltas in pv and mv back into in memory format +func ParseHLVFields(xattr []byte) (*HybridLogicalVector, error) { + var persistedHLV PersistedHLV + err := base.JSONUnmarshal(xattr, &persistedHLV) + if err != nil { + return nil, err + } + + pvMap, deltaErr := PersistedVVtoDeltas(persistedHLV.PreviousVersions) + if deltaErr != nil { + return nil, fmt.Errorf("error converting pv to deltas: %v", deltaErr) + } + mvMap, deltaErr := PersistedVVtoDeltas(persistedHLV.MergeVersions) + if deltaErr != nil { + return nil, fmt.Errorf("error converting mv to map: %v", deltaErr) + } + + newHLV := NewHybridLogicalVector() + newHLV.SourceID = persistedHLV.SourceID + newHLV.Version = persistedHLV.Version + newHLV.CurrentVersionCAS = persistedHLV.CurrentVersionCAS + newHLV.ImportCAS = persistedHLV.ImportCAS + newHLV.PreviousVersions = pvMap + newHLV.MergeVersions = mvMap + + return &newHLV, nil +} + // computeMetadataOnlyUpdate computes a new metadataOnlyUpdate based on the existing document's CAS and metadataOnlyUpdate func computeMetadataOnlyUpdate(currentCas uint64, revNo uint64, currentMou *MetadataOnlyUpdate) *MetadataOnlyUpdate { var prevCas string diff --git a/db/document_test.go b/db/document_test.go index 09dd07c97a..2648ea9140 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -239,18 +239,10 @@ const doc_meta_no_vv = `{ "time_saved": "2017-10-25T12:45:29.622450174-07:00" }` -const doc_meta_vv = `{ - "cvCas":"0x40e2010000000000", - "src":"cb06dc003846116d9b66d2ab23887a96", - "ver":"0x40e2010000000000", - "mv":{ - "s_LhRPsa7CpjEvP5zeXTXEBA":"c0ff05d7ac059a16", - "s_NqiIe0LekFPLeX4JvTO6Iw":"1c008cd6ac059a16" - }, - "pv":{ - "s_YZvBpEaztom9z5V/hDoeIw":"f0ff44d6ac059a16" - } - }` +const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", + "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","0x1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] +}` func TestParseVersionVectorSyncData(t *testing.T) { mv := make(HLVVersions) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 5b59b2f8d6..2286c0aa01 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -13,6 +13,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "sort" "strconv" "strings" @@ -30,6 +31,86 @@ type Version struct { Value uint64 `json:"version"` } +// VersionsDeltas will be sorted by version, first entry will be fill version then after that will be calculated deltas +type VersionsDeltas []DecodedVersion + +func (vde VersionsDeltas) Len() int { return len(vde) } + +func (vde VersionsDeltas) Swap(i, j int) { + vde[i], vde[j] = vde[j], vde[i] +} + +func (vde VersionsDeltas) Less(i, j int) bool { + if vde[i].Value == vde[j].Value { + return vde[i].SourceID < vde[j].SourceID + } + return vde[i].Value < vde[j].Value +} + +// VersionDeltas calculate the deltas of input map +func VersionDeltas(versions map[string]string) VersionsDeltas { + if versions == nil { + return nil + } + + vdm := make(VersionsDeltas, 0, len(versions)) + for src, vrs := range versions { + vdm = append(vdm, CreateDecodedVersion(src, base.HexCasToUint64(vrs))) + } + + // sort the list + sort.Sort(vdm) + + // traverse in reverse order and calculate delta between versions, leaving the first element as is + for i := len(vdm) - 1; i >= 1; i-- { + vdm[i].Value = vdm[i].Value - vdm[i-1].Value + } + return vdm +} + +// VersionsToDeltas will calculate deltas from the input map (pv or mv). Then will return the deltas in persisted format +func VersionsToDeltas(m map[string]string) []string { + if len(m) == 0 { + return nil + } + + var vrsList []string + deltas := VersionDeltas(m) + for _, delta := range deltas { + key := delta.SourceID + val := delta.Value + encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(val) + vrs := Version{SourceID: key, Value: encodedVal} + vrsList = append(vrsList, vrs.String()) + } + + return vrsList +} + +// PersistedVVtoDeltas converts the list of deltas in pv or mv from the bucket back from deltas into full versions in map format +func PersistedVVtoDeltas(vvList []string) (map[string]string, error) { + vv := make(map[string]string) + if len(vvList) == 0 { + return vv, nil + } + + var lastEntryVersion uint64 + for _, v := range vvList { + vrs, err := ParseVersion(v) + if err != nil { + return nil, err + } + ver, err := base.HexCasToUint64ForDelta([]byte(vrs.Value)) + if err != nil { + return nil, err + } + lastEntryVersion = ver + lastEntryVersion + calcVer := base.CasToString(lastEntryVersion) + vv[vrs.SourceID] = calcVer + } + return vv, nil +} + // CreateVersion creates an encoded sourceID and version pair func CreateVersion(source string, version uint64) Version { return Version{ @@ -88,6 +169,16 @@ type BucketVector struct { PreviousVersions map[string]string `json:"pv,omitempty"` } +// PersistedHLV is teh version of the version vector that is persisted to teh bucket, pv and mv will be lists of source version pairs in the bucket +type PersistedHLV struct { + CurrentVersionCAS string `json:"cvCas,omitempty"` // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication + ImportCAS string `json:"importCAS,omitempty"` // Set when an import modifies the document CAS but preserves the HLV (import of a version replicated by XDCR) + SourceID string `json:"src"` // source bucket uuid in (base64 encoded format) of where this entry originated from + Version string `json:"ver"` // current cas in little endian hex format of the current version on the version vector + MergeVersions []string `json:"mv,omitempty"` // list of merge versions in delta order. First elem will be full hex version, rest of items will be deltas calculated from the item above it + PreviousVersions []string `json:"pv,omitempty"` // list of previous versions in delta order. First elem will be full hex version, rest of items will be deltas calculated from the item above it +} + // NewHybridLogicalVector returns an initialised HybridLogicalVector. func NewHybridLogicalVector() HybridLogicalVector { return HybridLogicalVector{ diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index dacefd9506..a42ca3382e 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -10,10 +10,12 @@ package db import ( "encoding/base64" + "math/rand/v2" "reflect" "strconv" "strings" "testing" + "time" sgbucket "github.com/couchbase/sg-bucket" "github.com/couchbase/sync_gateway/base" @@ -162,6 +164,7 @@ func createHLVForTest(tb *testing.T, inputList []string) HybridLogicalVector { // first element will be current version and source pair currentVersionPair := strings.Split(inputList[0], "@") + // this needs changing hlvOutput.SourceID = base64.StdEncoding.EncodeToString([]byte(currentVersionPair[0])) value, err := strconv.ParseUint(currentVersionPair[1], 10, 64) require.NoError(tb, err) @@ -460,3 +463,88 @@ func TestParseCBLVersion(t *testing.T) { cblString := vrs.String() assert.Equal(t, vrsString, cblString) } + +// TestVersionDeltaCalculation: +// - Create some random versions and assign to a source/version map +// - Convert the map to deltas and assert that first item in list is greater than all other elements +// - Create a test HLV and convert it to persisted format in bytes +// - Convert this back to in memory format, assert each elem of in memory format previous versions map is the same as +// the corresponding element in the original pvMap +// - Do the same as above but for nil maps +func TestVersionDeltaCalculation(t *testing.T) { + src1 := "src1" + src2 := "src2" + src3 := "src3" + src4 := "src4" + src5 := "src5" + + timeNow := time.Now().UnixNano() + // make some version deltas + v1 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + v2 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + v3 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + v4 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + v5 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + + // make of source to delta + pvMap := make(map[string]string) + pvMap[src1] = v1 + pvMap[src2] = v2 + pvMap[src3] = v3 + pvMap[src4] = v4 + pvMap[src5] = v5 + + // convert to version delta map assert that first element is larger than all other elements + deltas := VersionDeltas(pvMap) + assert.Greater(t, deltas[0].Value, deltas[1].Value) + assert.Greater(t, deltas[0].Value, deltas[2].Value) + assert.Greater(t, deltas[0].Value, deltas[3].Value) + assert.Greater(t, deltas[0].Value, deltas[4].Value) + + // create a test hlv + inputHLVA := []string{"cluster3@2"} + hlv := createHLVForTest(t, inputHLVA) + hlv.PreviousVersions = pvMap + expSrc := hlv.SourceID + expVal := hlv.Version + expCas := hlv.CurrentVersionCAS + + // convert hlv to persisted format + vvXattr, err := ConstructXattrFromHlv(&hlv) + require.NoError(t, err) + + // convert the bytes back to an in memory format of hlv + memHLV, err := ParseHLVFields(vvXattr) + require.NoError(t, err) + + assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) + assert.Equal(t, pvMap[src2], memHLV.PreviousVersions[src2]) + assert.Equal(t, pvMap[src3], memHLV.PreviousVersions[src3]) + assert.Equal(t, pvMap[src4], memHLV.PreviousVersions[src4]) + assert.Equal(t, pvMap[src5], memHLV.PreviousVersions[src5]) + + // assert that the other elements are as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + + // test hlv with nil merge versions and nil previous versions to test panic safe + pvMap = nil + hlv2 := createHLVForTest(t, inputHLVA) + hlv2.PreviousVersions = pvMap + hlv2.MergeVersions = nil + deltas = VersionDeltas(pvMap) + assert.Nil(t, deltas) + // construct byte array from hlv + vvXattr, err = ConstructXattrFromHlv(&hlv2) + require.NoError(t, err) + // convert the bytes back to an in memory format of hlv + memHLV, err = ParseHLVFields(vvXattr) + require.NoError(t, err) + // assert in memory hlv is as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.PreviousVersions, 0) + assert.Len(t, memHLV.MergeVersions, 0) +} diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go index 477e82dab2..1edd02ace4 100644 --- a/db/utilities_hlv_testing.go +++ b/db/utilities_hlv_testing.go @@ -63,6 +63,27 @@ func (h *HLVAgent) InsertWithHLV(ctx context.Context, key string) (casOut uint64 return cas } +// UpdateWithHLV will update and existing doc in bucket mocking write from another hlv aware peer +func (h *HLVAgent) UpdateWithHLV(ctx context.Context, key string, inputCas uint64, hlv *HybridLogicalVector) (casOut uint64) { + err := hlv.AddVersion(CreateVersion(h.Source, hlvExpandMacroCASValue)) + require.NoError(h.t, err) + hlv.CurrentVersionCAS = hlvExpandMacroCASValue + + vvXattr, err := ConstructXattrFromHlv(hlv) + require.NoError(h.t, err) + mutateInOpts := &sgbucket.MutateInOptions{ + MacroExpansion: hlv.computeMacroExpansions(), + } + + docBody := base.MustJSONMarshal(h.t, defaultHelperBody) + xattrData := map[string][]byte{ + h.xattrName: vvXattr, + } + cas, err := h.datastore.WriteWithXattrs(ctx, key, 0, inputCas, docBody, xattrData, nil, mutateInOpts) + require.NoError(h.t, err) + return cas +} + // EncodeTestVersion converts a simplified string version of the form 1@abc to a hex-encoded version and base64 encoded // source, like 169a05acd705ffc0@YWJj. Allows use of simplified versions in tests for readability, ease of use. func EncodeTestVersion(versionString string) (encodedString string) { diff --git a/go.sum b/go.sum index f7440db3b7..6a62927c63 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,13 @@ github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/ github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/blance v0.1.6 h1:zyNew/SN2AheIoJxQ2LqqA1u3bMp03eGCer6hSDMUDs= -github.com/couchbase/blance v0.1.6/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= -github.com/couchbase/cbauth v0.1.12 h1:JOAWjjp2BdubvrrggvN4yQo3oEc2ndXcRN1ONCklUOM= +github.com/couchbase/blance v0.1.5 h1:kNSAwhb8FXSJpicJ8R8Kk7+0V1+MyTcY1MOHIDbU79w= +github.com/couchbase/blance v0.1.5/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= +github.com/couchbase/cbauth v0.1.11 h1:LLyGiVnsKxyHp9wbOQk87oF9eDUSh1in2vh/l6vaezg= +github.com/couchbase/cbauth v0.1.11/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= github.com/couchbase/cbauth v0.1.12/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= -github.com/couchbase/cbgt v1.4.1 h1:lJtZTrPkbzq1FXRFdd6pGRCBtEL1/VIH8pWQXLTxZgI= -github.com/couchbase/cbgt v1.4.1/go.mod h1:QR8XIUzSm2cFviBkdBCdpa87M2oe5yMVIzvsJGm/BUI= +github.com/couchbase/cbgt v1.3.9 h1:MAT3FwD1ctekxuFe0yau0H1BCTvgLXvh1ipbZ3nZhBE= +github.com/couchbase/cbgt v1.3.9/go.mod h1:MImhtmvk0qjJit5HbmA34tnYThZoNtvgjL7jJH/kCAE= github.com/couchbase/clog v0.1.0 h1:4Kh/YHkhRjMCbdQuvRVsm39XZh4FtL1d8fAwJsHrEPY= github.com/couchbase/clog v0.1.0/go.mod h1:7tzUpEOsE+fgU81yfcjy5N1H6XtbVC8SgOz/3mCjmd4= github.com/couchbase/go-blip v0.0.0-20231212195435-3490e96d30e3 h1:MeikDkvUMHZLpS57pfzhu2E+disqUVulUTb/r3aqUck= @@ -56,10 +57,14 @@ github.com/couchbase/gocbcoreps v0.1.3 h1:fILaKGCjxFIeCgAUG8FGmRDSpdrRggohOMKEgO github.com/couchbase/gocbcoreps v0.1.3/go.mod h1:hBFpDNPnRno6HH5cRXExhqXYRmTsFJlFHQx7vztcXPk= github.com/couchbase/gomemcached v0.2.1 h1:lDONROGbklo8pOt4Sr4eV436PVEaKDr3o9gUlhv9I2U= github.com/couchbase/gomemcached v0.2.1/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= +github.com/couchbase/gomemcached v0.2.2-0.20230407174933-7d7ce13da8cc h1:OORlVS7wk2+Yp0GuUvtlGY2g0h4sTtIZG6DYOF2Udro= +github.com/couchbase/gomemcached v0.2.2-0.20230407174933-7d7ce13da8cc/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= github.com/couchbase/goprotostellar v1.0.2 h1:yoPbAL9sCtcyZ5e/DcU5PRMOEFaJrF9awXYu3VPfGls= github.com/couchbase/goprotostellar v1.0.2/go.mod h1:5/yqVnZlW2/NSbAWu1hPJCFBEwjxgpe0PFFOlRixnp4= github.com/couchbase/goutils v0.1.2 h1:gWr8B6XNWPIhfalHNog3qQKfGiYyh4K4VhO3P2o9BCs= github.com/couchbase/goutils v0.1.2/go.mod h1:h89Ek/tiOxxqjz30nPPlwZdQbdB8BwgnuBxeoUe/ViE= +github.com/couchbase/goxdcr v0.0.0-20240815203157-dfba7a5b4251 h1:sfRx1INRG/dMOTwM+sNWvuqZWQ7XPll1OG1Bgyz6sAc= +github.com/couchbase/goxdcr v0.0.0-20240815203157-dfba7a5b4251/go.mod h1:2V0ptUlrXzFPpT+sti3VtNBWesryr2fG9yGXYKakyjI= github.com/couchbase/sg-bucket v0.0.0-20240606153601-d152b90edccb h1:FrUz2LZLmTwQl1cRCXUDwouE3gINsaEAV4o6BdAftz8= github.com/couchbase/sg-bucket v0.0.0-20240606153601-d152b90edccb/go.mod h1:IQisEdcLRfS/pjSgmqG/8gerVm0Q7GrvpQtMIZ7oYt4= github.com/couchbase/tools-common/cloud v1.0.0 h1:SQZIccXoedbrThehc/r9BJbpi/JhwJ8X00PDjZ2gEBE= diff --git a/rest/api_test.go b/rest/api_test.go index f8f58b5f2e..b02cc645f7 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -138,6 +138,19 @@ func TestPublicRESTStatCount(t *testing.T) { }, 2) } +func TestGreg(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + + rt.PutDoc("doc", `{"greg":"test"}`) + fmt.Println("here") + + rt.GetDatabase().FlushRevisionCacheForTest() + + _, bod := rt.GetDoc("doc") + fmt.Println(bod) +} + func TestDBRoot(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{GuestEnabled: true}) defer rt.Close() @@ -2801,6 +2814,62 @@ func TestCreateDBWithXattrsDisbaled(t *testing.T) { assert.Contains(t, resp.Body.String(), errResp) } +// TestPvDeltaReadAndWrite: +// - Write a doc from another hlv aware peer to the bucket +// - Force import of this doc, then update this doc via rest tester source +// - Assert that the document hlv is as expected +// - Update the doc from a new hlv aware peer and force the import of this new write +// - Asser that the new hlv is as expected, testing that the hlv went through transformation to the persisted delta +// version and back to the in memory version as expected +func TestPvDeltaReadAndWrite(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() + testSource := rt.GetDatabase().EncodedSourceID + + const docID = "doc1" + otherSource := "otherSource" + hlvHelper := db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") + existingHLVKey := docID + cas := hlvHelper.InsertWithHLV(ctx, existingHLVKey) + encodedCASV1 := db.EncodeValue(cas) + encodedSourceV1 := db.EncodeSource(otherSource) + + // force import of this write + version1, _ := rt.GetDoc(docID) + + // update the above doc, this should push CV to PV and adds a new CV + version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"new": "update!"}) + newDoc, _, err := collection.GetDocWithXattrs(ctx, existingHLVKey, db.DocUnmarshalAll) + require.NoError(t, err) + encodedCASV2 := db.EncodeValue(newDoc.Cas) + encodedSourceV2 := testSource + + // assert that we have a prev CV drop to pv and a new CV pair, assert pv values are as expected after delta conversions + assert.Equal(t, testSource, newDoc.HLV.SourceID) + assert.Equal(t, version2.CV.Value, newDoc.HLV.Version) + assert.Len(t, newDoc.HLV.PreviousVersions, 1) + assert.Equal(t, encodedCASV1, newDoc.HLV.PreviousVersions[encodedSourceV1]) + + otherSource = "diffSource" + hlvHelper = db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") + cas = hlvHelper.UpdateWithHLV(ctx, existingHLVKey, newDoc.Cas, newDoc.HLV) + encodedSourceV3 := db.EncodeSource(otherSource) + encodedCASV3 := db.EncodeValue(cas) + + // import and get raw doc + _, _ = rt.GetDoc(docID) + bucketDoc, _, err := collection.GetDocWithXattrs(ctx, docID, db.DocUnmarshalAll) + require.NoError(t, err) + + // assert that we have two entries in previous versions, and they are correctly converted from deltas back to full value + assert.Equal(t, encodedSourceV3, bucketDoc.HLV.SourceID) + assert.Equal(t, encodedCASV3, bucketDoc.HLV.Version) + assert.Len(t, bucketDoc.HLV.PreviousVersions, 2) + assert.Equal(t, encodedCASV1, bucketDoc.HLV.PreviousVersions[encodedSourceV1]) + assert.Equal(t, encodedCASV2, bucketDoc.HLV.PreviousVersions[encodedSourceV2]) +} + // TestPutDocUpdateVersionVector: // - Put a doc and assert that the versions and the source for the hlv is correctly updated // - Update that doc and assert HLV has also been updated From 1d2adc1b53dc1717d1800bb7899c5e624cd2996e Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 29 Aug 2024 15:31:50 +0100 Subject: [PATCH 02/19] tidy up --- base/constants.go | 10 ---------- db/document.go | 6 +++--- db/hybrid_logical_vector.go | 6 +++--- db/hybrid_logical_vector_test.go | 2 +- go.sum | 15 +++++---------- rest/api_test.go | 15 +-------------- 6 files changed, 13 insertions(+), 41 deletions(-) diff --git a/base/constants.go b/base/constants.go index bb5aada41a..49530a20ac 100644 --- a/base/constants.go +++ b/base/constants.go @@ -184,16 +184,6 @@ const ( SyncFnErrorMissingChannelAccess = "sg missing channel access" ) -const ( - HLV_CVCAS_FIELD = "cvCas" // This stores the version of the HLV - // These are the fields in the HLV - HLV_SRC_FIELD = "src" // source id for HLV - HLV_VER_FIELD = "ver" // ver field in HLV - HLV_MV_FIELD = "mv" // the MV field in HLV - HLV_PV_FIELD = "pv" // The PV field HLV - HLV_IMPORT_CAS = "importCAS" // import cas field -) - const ( // EmptyDocument denotes an empty document in JSON form. EmptyDocument = `{}` diff --git a/db/document.go b/db/document.go index 1f92999bd3..2d8a193025 100644 --- a/db/document.go +++ b/db/document.go @@ -1325,11 +1325,11 @@ func ParseHLVFields(xattr []byte) (*HybridLogicalVector, error) { return nil, err } - pvMap, deltaErr := PersistedVVtoDeltas(persistedHLV.PreviousVersions) + pvMap, deltaErr := PersistedDeltasToMap(persistedHLV.PreviousVersions) if deltaErr != nil { - return nil, fmt.Errorf("error converting pv to deltas: %v", deltaErr) + return nil, fmt.Errorf("error converting pv to map: %v", deltaErr) } - mvMap, deltaErr := PersistedVVtoDeltas(persistedHLV.MergeVersions) + mvMap, deltaErr := PersistedDeltasToMap(persistedHLV.MergeVersions) if deltaErr != nil { return nil, fmt.Errorf("error converting mv to map: %v", deltaErr) } diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 2286c0aa01..bda0424f1b 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -87,8 +87,8 @@ func VersionsToDeltas(m map[string]string) []string { return vrsList } -// PersistedVVtoDeltas converts the list of deltas in pv or mv from the bucket back from deltas into full versions in map format -func PersistedVVtoDeltas(vvList []string) (map[string]string, error) { +// PersistedDeltasToMap converts the list of deltas in pv or mv from the bucket back from deltas into full versions in map format +func PersistedDeltasToMap(vvList []string) (map[string]string, error) { vv := make(map[string]string) if len(vvList) == 0 { return vv, nil @@ -169,7 +169,7 @@ type BucketVector struct { PreviousVersions map[string]string `json:"pv,omitempty"` } -// PersistedHLV is teh version of the version vector that is persisted to teh bucket, pv and mv will be lists of source version pairs in the bucket +// PersistedHLV is the version of the version vector that is persisted to the bucket, pv and mv will be lists of source version pairs in the bucket type PersistedHLV struct { CurrentVersionCAS string `json:"cvCas,omitempty"` // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication ImportCAS string `json:"importCAS,omitempty"` // Set when an import modifies the document CAS but preserves the HLV (import of a version replicated by XDCR) diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index a42ca3382e..b695237fe7 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -486,7 +486,7 @@ func TestVersionDeltaCalculation(t *testing.T) { v4 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) v5 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) - // make of source to delta + // make map of source to version pvMap := make(map[string]string) pvMap[src1] = v1 pvMap[src2] = v2 diff --git a/go.sum b/go.sum index 6a62927c63..f7440db3b7 100644 --- a/go.sum +++ b/go.sum @@ -36,13 +36,12 @@ github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/ github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/blance v0.1.5 h1:kNSAwhb8FXSJpicJ8R8Kk7+0V1+MyTcY1MOHIDbU79w= -github.com/couchbase/blance v0.1.5/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= -github.com/couchbase/cbauth v0.1.11 h1:LLyGiVnsKxyHp9wbOQk87oF9eDUSh1in2vh/l6vaezg= -github.com/couchbase/cbauth v0.1.11/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= +github.com/couchbase/blance v0.1.6 h1:zyNew/SN2AheIoJxQ2LqqA1u3bMp03eGCer6hSDMUDs= +github.com/couchbase/blance v0.1.6/go.mod h1:2Sa/nsJSieN/r3T9LsrUYWeQ015qDsuHybhz4F4JcHU= +github.com/couchbase/cbauth v0.1.12 h1:JOAWjjp2BdubvrrggvN4yQo3oEc2ndXcRN1ONCklUOM= github.com/couchbase/cbauth v0.1.12/go.mod h1:W7zkNXa0B2cTDg90YmmuTSbu+PlYOvMqzQvmNlNH/Mg= -github.com/couchbase/cbgt v1.3.9 h1:MAT3FwD1ctekxuFe0yau0H1BCTvgLXvh1ipbZ3nZhBE= -github.com/couchbase/cbgt v1.3.9/go.mod h1:MImhtmvk0qjJit5HbmA34tnYThZoNtvgjL7jJH/kCAE= +github.com/couchbase/cbgt v1.4.1 h1:lJtZTrPkbzq1FXRFdd6pGRCBtEL1/VIH8pWQXLTxZgI= +github.com/couchbase/cbgt v1.4.1/go.mod h1:QR8XIUzSm2cFviBkdBCdpa87M2oe5yMVIzvsJGm/BUI= github.com/couchbase/clog v0.1.0 h1:4Kh/YHkhRjMCbdQuvRVsm39XZh4FtL1d8fAwJsHrEPY= github.com/couchbase/clog v0.1.0/go.mod h1:7tzUpEOsE+fgU81yfcjy5N1H6XtbVC8SgOz/3mCjmd4= github.com/couchbase/go-blip v0.0.0-20231212195435-3490e96d30e3 h1:MeikDkvUMHZLpS57pfzhu2E+disqUVulUTb/r3aqUck= @@ -57,14 +56,10 @@ github.com/couchbase/gocbcoreps v0.1.3 h1:fILaKGCjxFIeCgAUG8FGmRDSpdrRggohOMKEgO github.com/couchbase/gocbcoreps v0.1.3/go.mod h1:hBFpDNPnRno6HH5cRXExhqXYRmTsFJlFHQx7vztcXPk= github.com/couchbase/gomemcached v0.2.1 h1:lDONROGbklo8pOt4Sr4eV436PVEaKDr3o9gUlhv9I2U= github.com/couchbase/gomemcached v0.2.1/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= -github.com/couchbase/gomemcached v0.2.2-0.20230407174933-7d7ce13da8cc h1:OORlVS7wk2+Yp0GuUvtlGY2g0h4sTtIZG6DYOF2Udro= -github.com/couchbase/gomemcached v0.2.2-0.20230407174933-7d7ce13da8cc/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= github.com/couchbase/goprotostellar v1.0.2 h1:yoPbAL9sCtcyZ5e/DcU5PRMOEFaJrF9awXYu3VPfGls= github.com/couchbase/goprotostellar v1.0.2/go.mod h1:5/yqVnZlW2/NSbAWu1hPJCFBEwjxgpe0PFFOlRixnp4= github.com/couchbase/goutils v0.1.2 h1:gWr8B6XNWPIhfalHNog3qQKfGiYyh4K4VhO3P2o9BCs= github.com/couchbase/goutils v0.1.2/go.mod h1:h89Ek/tiOxxqjz30nPPlwZdQbdB8BwgnuBxeoUe/ViE= -github.com/couchbase/goxdcr v0.0.0-20240815203157-dfba7a5b4251 h1:sfRx1INRG/dMOTwM+sNWvuqZWQ7XPll1OG1Bgyz6sAc= -github.com/couchbase/goxdcr v0.0.0-20240815203157-dfba7a5b4251/go.mod h1:2V0ptUlrXzFPpT+sti3VtNBWesryr2fG9yGXYKakyjI= github.com/couchbase/sg-bucket v0.0.0-20240606153601-d152b90edccb h1:FrUz2LZLmTwQl1cRCXUDwouE3gINsaEAV4o6BdAftz8= github.com/couchbase/sg-bucket v0.0.0-20240606153601-d152b90edccb/go.mod h1:IQisEdcLRfS/pjSgmqG/8gerVm0Q7GrvpQtMIZ7oYt4= github.com/couchbase/tools-common/cloud v1.0.0 h1:SQZIccXoedbrThehc/r9BJbpi/JhwJ8X00PDjZ2gEBE= diff --git a/rest/api_test.go b/rest/api_test.go index b02cc645f7..4ef8cc0bf1 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -138,19 +138,6 @@ func TestPublicRESTStatCount(t *testing.T) { }, 2) } -func TestGreg(t *testing.T) { - rt := NewRestTester(t, nil) - defer rt.Close() - - rt.PutDoc("doc", `{"greg":"test"}`) - fmt.Println("here") - - rt.GetDatabase().FlushRevisionCacheForTest() - - _, bod := rt.GetDoc("doc") - fmt.Println(bod) -} - func TestDBRoot(t *testing.T) { rt := NewRestTester(t, &RestTesterConfig{GuestEnabled: true}) defer rt.Close() @@ -2819,7 +2806,7 @@ func TestCreateDBWithXattrsDisbaled(t *testing.T) { // - Force import of this doc, then update this doc via rest tester source // - Assert that the document hlv is as expected // - Update the doc from a new hlv aware peer and force the import of this new write -// - Asser that the new hlv is as expected, testing that the hlv went through transformation to the persisted delta +// - Assert that the new hlv is as expected, testing that the hlv went through transformation to the persisted delta // version and back to the in memory version as expected func TestPvDeltaReadAndWrite(t *testing.T) { rt := NewRestTester(t, nil) From f3c6a7ddc1719069cbd932e5b6cb5fc2b9fee4a9 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 29 Aug 2024 15:42:59 +0100 Subject: [PATCH 03/19] fix lint --- db/document.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/document.go b/db/document.go index 2d8a193025..f151fbae30 100644 --- a/db/document.go +++ b/db/document.go @@ -1292,7 +1292,7 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl return data, syncXattr, vvXattr, mouXattr, globalXattr, nil } -// ConstructXattrFromHlv will build a persisted hlv from teh in memory hlv. Converting the pv and mv maps to deltas +// ConstructXattrFromHlv will build a persisted hlv from the in memory hlv. Converting the pv and mv maps to deltas func ConstructXattrFromHlv(hlv *HybridLogicalVector) ([]byte, error) { var persistedHLV PersistedHLV From f784390c6056f45de5ada130a3f89f00a1a53e61 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Mon, 2 Sep 2024 13:27:19 +0100 Subject: [PATCH 04/19] remove comment --- db/hybrid_logical_vector_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index b695237fe7..bfed01d474 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -164,7 +164,6 @@ func createHLVForTest(tb *testing.T, inputList []string) HybridLogicalVector { // first element will be current version and source pair currentVersionPair := strings.Split(inputList[0], "@") - // this needs changing hlvOutput.SourceID = base64.StdEncoding.EncodeToString([]byte(currentVersionPair[0])) value, err := strconv.ParseUint(currentVersionPair[1], 10, 64) require.NoError(tb, err) From 75e95744384afb043abadb6abae574c8b9b93e74 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 5 Sep 2024 10:15:11 +0100 Subject: [PATCH 05/19] changes to reflect xdcr format --- db/blip_handler.go | 2 +- db/blip_sync_context.go | 2 +- db/crud.go | 8 ++++---- db/database_test.go | 8 ++++---- db/document_test.go | 4 ++-- db/hybrid_logical_vector.go | 20 +++++++++++++------- db/utilities_hlv_testing.go | 6 +++--- rest/utilities_testing.go | 2 +- rest/utilities_testing_blip_client.go | 10 +++++----- 9 files changed, 34 insertions(+), 28 deletions(-) diff --git a/db/blip_handler.go b/db/blip_handler.go index a5f25ea5a8..942e30613b 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -515,7 +515,7 @@ func (bh *blipHandler) sendChanges(sender *blip.Sender, opts *sendChangesOptions pendingChanges = append(pendingChanges, changeRow) } } else { - changeRow := bh.buildChangesRow(change, change.CurrentVersion.String()) + changeRow := bh.buildChangesRow(change, change.CurrentVersion.BlipString()) pendingChanges = append(pendingChanges, changeRow) } diff --git a/db/blip_sync_context.go b/db/blip_sync_context.go index 584cfc496b..5994e22cef 100644 --- a/db/blip_sync_context.go +++ b/db/blip_sync_context.go @@ -631,7 +631,7 @@ func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sende docRev, originalErr = handleChangesResponseCollection.GetRev(ctx, docID, revID, true, nil) } else { // extract CV string rev representation - version, vrsErr := ParseVersion(revID) + version, vrsErr := ParseBlipVersion(revID) if vrsErr != nil { return vrsErr } diff --git a/db/crud.go b/db/crud.go index 45221aaa83..1294ce74c1 100644 --- a/db/crud.go +++ b/db/crud.go @@ -345,7 +345,7 @@ func (db *DatabaseCollectionWithUser) documentRevisionForRequest(ctx context.Con if revID != nil { requestedVersion = *revID } else if cv != nil { - requestedVersion = cv.String() + requestedVersion = cv.BlipString() } if revision.BodyBytes == nil { @@ -3059,7 +3059,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedRev(ctx context.Context, doci // CheckProposedVersion - given DocID and a version in string form, check whether it can be added without conflict. func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, docid, proposedVersionStr string, previousVersionStr string) (status ProposedRevStatus, currentVersion string) { - proposedVersion, err := ParseVersion(proposedVersionStr) + proposedVersion, err := ParseBlipVersion(proposedVersionStr) if err != nil { base.WarnfCtx(ctx, "Couldn't parse proposed version for doc %q / %q: %v", base.UD(docid), proposedVersionStr, err) return ProposedRev_Error, "" @@ -3068,7 +3068,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, var previousVersion Version if previousVersionStr != "" { var err error - previousVersion, err = ParseVersion(previousVersionStr) + previousVersion, err = ParseBlipVersion(previousVersionStr) if err != nil { base.WarnfCtx(ctx, "Couldn't parse previous version for doc %q / %q: %v", base.UD(docid), previousVersionStr, err) return ProposedRev_Error, "" @@ -3100,7 +3100,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, // Conflict, return the current cv. This may be a false positive conflict if the client has replicated // the server cv via a different peer. Client is responsible for performing this check based on the // returned localDocCV - return ProposedRev_Conflict, localDocCV.String() + return ProposedRev_Conflict, localDocCV.BlipString() } } diff --git a/db/database_test.go b/db/database_test.go index bed5a7ec58..a9002dfa1c 100644 --- a/db/database_test.go +++ b/db/database_test.go @@ -391,7 +391,7 @@ func TestCheckProposedVersion(t *testing.T) { newVersion: Version{"other", incrementCas(cvValue, -100)}, previousVersion: &Version{cvSource, incrementCas(cvValue, -50)}, expectedStatus: ProposedRev_Conflict, - expectedRev: Version{cvSource, cvValue}.String(), + expectedRev: Version{cvSource, cvValue}.BlipString(), }, { // conflict - previous version is older than CV @@ -399,7 +399,7 @@ func TestCheckProposedVersion(t *testing.T) { newVersion: Version{"other", incrementCas(cvValue, 100)}, previousVersion: &Version{"other", incrementCas(cvValue, -50)}, expectedStatus: ProposedRev_Conflict, - expectedRev: Version{cvSource, cvValue}.String(), + expectedRev: Version{cvSource, cvValue}.BlipString(), }, } @@ -407,9 +407,9 @@ func TestCheckProposedVersion(t *testing.T) { t.Run(tc.name, func(t *testing.T) { previousVersionStr := "" if tc.previousVersion != nil { - previousVersionStr = tc.previousVersion.String() + previousVersionStr = tc.previousVersion.BlipString() } - status, rev := collection.CheckProposedVersion(ctx, "doc1", tc.newVersion.String(), previousVersionStr) + status, rev := collection.CheckProposedVersion(ctx, "doc1", tc.newVersion.BlipString(), previousVersionStr) assert.Equal(t, tc.expectedStatus, status) assert.Equal(t, tc.expectedRev, rev) }) diff --git a/db/document_test.go b/db/document_test.go index 2648ea9140..2659fa588f 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -240,8 +240,8 @@ const doc_meta_no_vv = `{ }` const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", - "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","0x1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], - "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] + "mv":["s_LhRPsa7CpjEvP5zeXTXEBA@0xc0ff05d7ac059a16","s_NqiIe0LekFPLeX4JvTO6Iw@0x1c008cd6"], + "pv":["s_YZvBpEaztom9z5V/hDoeIw@0xf0ff44d6ac059a16"] }` func TestParseVersionVectorSyncData(t *testing.T) { diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index bda0424f1b..8cd81d99ef 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -81,7 +81,7 @@ func VersionsToDeltas(m map[string]string) []string { val := delta.Value encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(val) vrs := Version{SourceID: key, Value: encodedVal} - vrsList = append(vrsList, vrs.String()) + vrsList = append(vrsList, vrs.DeltaString()) } return vrsList @@ -96,7 +96,7 @@ func PersistedDeltasToMap(vvList []string) (map[string]string, error) { var lastEntryVersion uint64 for _, v := range vvList { - vrs, err := ParseVersion(v) + vrs, err := ParseDeltasVersion(v) if err != nil { return nil, err } @@ -119,7 +119,8 @@ func CreateVersion(source string, version uint64) Version { } } -func ParseVersion(versionString string) (version Version, err error) { +// ParseBlipVersion will parse source version pair from blip format +func ParseBlipVersion(versionString string) (version Version, err error) { timestampString, sourceBase64, found := strings.Cut(versionString, "@") if !found { return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString) @@ -142,6 +143,11 @@ func (v Version) String() string { return strconv.FormatUint(v.Value, 16) + "@" + v.SourceID } +// DeltaString returns a pv or mv delta format string representation of version +func (v Version) DeltaString() string { + return v.SourceID + "@" + v.Value +} + // ExtractCurrentVersionFromHLV will take the current version form the HLV struct and return it in the Version struct func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { src, vrs := hlv.GetCurrentVersion() @@ -201,7 +207,7 @@ func (hlv *HybridLogicalVector) GetCurrentVersionString() string { SourceID: hlv.SourceID, Value: hlv.Version, } - return version.String() + return version.BlipString() } // IsVersionKnown checks to see whether the HLV already contains a Version for the provided @@ -384,7 +390,7 @@ func (hlv *HybridLogicalVector) ToHistoryForHLV() string { itemNo := 1 for key, value := range hlv.MergeVersions { vrs := Version{SourceID: key, Value: value} - s.WriteString(vrs.String()) + s.WriteString(vrs.BlipString()) if itemNo < len(hlv.MergeVersions) { s.WriteString(",") } @@ -400,7 +406,7 @@ func (hlv *HybridLogicalVector) ToHistoryForHLV() string { } for key, value := range hlv.PreviousVersions { vrs := Version{SourceID: key, Value: value} - s.WriteString(vrs.String()) + s.WriteString(vrs.BlipString()) if itemNo < len(hlv.PreviousVersions) { s.WriteString(",") } @@ -505,7 +511,7 @@ func parseVectorValues(vectorStr string) (versions []Version, err error) { if len(v) > 0 && v[0] == ' ' { v = v[1:] } - version, err := ParseVersion(v) + version, err := ParseBlipVersion(v) if err != nil { return nil, err } diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go index 1edd02ace4..4c6e9f9a2e 100644 --- a/db/utilities_hlv_testing.go +++ b/db/utilities_hlv_testing.go @@ -150,7 +150,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // pv for _, versionStr := range strings.Split(pvString, ",") { - version, err := ParseVersion(versionStr) + version, err := ParseBlipVersion(versionStr) require.NoError(t, err) pv[EncodeSource(version.SourceID)] = version.Value } @@ -158,7 +158,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // mv if mvString != "" { for _, versionStr := range strings.Split(mvString, ",") { - version, err := ParseVersion(versionStr) + version, err := ParseBlipVersion(versionStr) require.NoError(t, err) mv[EncodeSource(version.SourceID)] = version.Value } @@ -168,7 +168,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // Requires that the CV for the provided HLV matches the expected CV (sent in simplified test format) func RequireCVEqual(t *testing.T, hlv *HybridLogicalVector, expectedCV string) { - testVersion, err := ParseVersion(expectedCV) + testVersion, err := ParseBlipVersion(expectedCV) require.NoError(t, err) require.Equal(t, EncodeSource(testVersion.SourceID), hlv.SourceID) require.Equal(t, testVersion.Value, hlv.Version) diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 108ee0d2fa..625f5c2496 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -2468,7 +2468,7 @@ func (v DocVersion) GetRev(useHLV bool) string { if v.CV.SourceID == "" { return "" } - return v.CV.String() + return v.CV.BlipString() } else { return v.RevTreeID } diff --git a/rest/utilities_testing_blip_client.go b/rest/utilities_testing_blip_client.go index d433d96ebc..c5047ef05d 100644 --- a/rest/utilities_testing_blip_client.go +++ b/rest/utilities_testing_blip_client.go @@ -175,7 +175,7 @@ func (btcr *BlipTesterCollectionClient) NewBlipTesterDoc(revID string, body []by } func VersionFromRevID(revID string) db.Version { - version, err := db.ParseVersion(revID) + version, err := db.ParseBlipVersion(revID) if err != nil { panic(err) } @@ -924,7 +924,7 @@ func (btc *BlipTesterCollectionClient) PushRev(docID string, parentVersion DocVe func (btc *BlipTesterCollectionClient) requireRevID(expected DocVersion, revID string) { if btc.UseHLV() { - require.Equal(btc.parent.rt.TB(), expected.CV.String(), revID) + require.Equal(btc.parent.rt.TB(), expected.CV.BlipString(), revID) } else { require.Equal(btc.parent.rt.TB(), expected.RevTreeID, revID) } @@ -1193,7 +1193,7 @@ func (btc *BlipTesterClient) AssertOnBlipHistory(t *testing.T, msg *blip.Message require.NoError(t, err) if subProtocol >= db.CBMobileReplicationV4 { // history could be empty a lot of the time in HLV messages as updates from the same source won't populate previous versions if msg.Properties[db.RevMessageHistory] != "" { - assert.Equal(t, docVersion.CV.String(), msg.Properties[db.RevMessageHistory]) + assert.Equal(t, docVersion.CV.BlipString(), msg.Properties[db.RevMessageHistory]) } } else { assert.Equal(t, docVersion.RevTreeID, msg.Properties[db.RevMessageHistory]) @@ -1207,7 +1207,7 @@ func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVe if doc, ok := btc.docs[docID]; ok { if doc.revMode == revModeHLV { - if doc.getCurrentRevID() == docVersion.CV.String() { + if doc.getCurrentRevID() == docVersion.CV.BlipString() { return doc.body, true } } else { @@ -1306,7 +1306,7 @@ func (btr *BlipTesterReplicator) storeMessage(msg *blip.Message) { func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, version DocVersion) (msg *blip.Message) { var revID string if btc.UseHLV() { - revID = version.CV.String() + revID = version.CV.BlipString() } else { revID = version.RevTreeID } From 0e5f537f7b01c7c18d99b0c14b6e53fe9c498bb1 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 12 Sep 2024 11:25:09 +0100 Subject: [PATCH 06/19] updates based off review --- base/util.go | 6 ++--- base/util_test.go | 16 ++++++++--- db/blip_handler.go | 2 +- db/blip_sync_context.go | 2 +- db/change_cache.go | 2 +- db/crud.go | 8 +++--- db/database_test.go | 8 +++--- db/document.go | 14 +++++----- db/document_test.go | 35 ++++++++++++++++++++++--- db/hybrid_logical_vector.go | 21 ++++++--------- db/hybrid_logical_vector_test.go | 4 +-- db/import_listener.go | 2 +- db/utilities_hlv_testing.go | 6 ++--- rest/importuserxattrtest/import_test.go | 6 ++--- rest/utilities_testing.go | 2 +- rest/utilities_testing_blip_client.go | 10 +++---- 16 files changed, 87 insertions(+), 57 deletions(-) diff --git a/base/util.go b/base/util.go index 5afa8a3b66..89fcb7f572 100644 --- a/base/util.go +++ b/base/util.go @@ -48,8 +48,6 @@ const ( kMaxDeltaTtlDuration = 60 * 60 * 24 * 30 * time.Second ) -var MaxDecodedLength = len(Uint64CASToLittleEndianHex(math.MaxUint64)) - // NonCancellableContext is here to stroe a context that is not cancellable. Used to explicitly state when a change from // a cancellable context to a context withoutr contex is required type NonCancellableContext struct { @@ -1037,7 +1035,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { casByte = evenHexLen } - decoded := make([]byte, MaxDecodedLength) + decoded := make([]byte, 8) _, err := hex.Decode(decoded, casByte[2:]) if err != nil { return 0, err @@ -1046,7 +1044,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { return res, nil } -// Uint64ToLittleEndianHexAndStripZeros will convert a uin64 type to little endian hex, stripping any zeros off the end +// Uint64ToLittleEndianHexAndStripZeros will convert a uint64 type to little endian hex, stripping any zeros off the end func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string { hexCas := Uint64CASToLittleEndianHex(cas) diff --git a/base/util_test.go b/base/util_test.go index b37c67f647..51c40c65e8 100644 --- a/base/util_test.go +++ b/base/util_test.go @@ -1740,18 +1740,28 @@ func TestUint64CASToLittleEndianHexAndStripZeros(t *testing.T) { hexLE := "0x0000000000000000" u64 := HexCasToUint64(hexLE) hexLEStripped := Uint64ToLittleEndianHexAndStripZeros(u64) - u64Stripped := HexCasToUint64(hexLEStripped) + u64Stripped, err := HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) assert.Equal(t, u64, u64Stripped) hexLE = "0xffffffffffffffff" u64 = HexCasToUint64(hexLE) hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) - u64Stripped = HexCasToUint64(hexLEStripped) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) assert.Equal(t, u64, u64Stripped) hexLE = "0xd123456e789a0bcf" u64 = HexCasToUint64(hexLE) hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) - u64Stripped = HexCasToUint64(hexLEStripped) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) + + hexLE = "0xd123456e78000000" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) assert.Equal(t, u64, u64Stripped) } diff --git a/db/blip_handler.go b/db/blip_handler.go index 942e30613b..a5f25ea5a8 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -515,7 +515,7 @@ func (bh *blipHandler) sendChanges(sender *blip.Sender, opts *sendChangesOptions pendingChanges = append(pendingChanges, changeRow) } } else { - changeRow := bh.buildChangesRow(change, change.CurrentVersion.BlipString()) + changeRow := bh.buildChangesRow(change, change.CurrentVersion.String()) pendingChanges = append(pendingChanges, changeRow) } diff --git a/db/blip_sync_context.go b/db/blip_sync_context.go index 5994e22cef..584cfc496b 100644 --- a/db/blip_sync_context.go +++ b/db/blip_sync_context.go @@ -631,7 +631,7 @@ func (bsc *BlipSyncContext) sendRevision(ctx context.Context, sender *blip.Sende docRev, originalErr = handleChangesResponseCollection.GetRev(ctx, docID, revID, true, nil) } else { // extract CV string rev representation - version, vrsErr := ParseBlipVersion(revID) + version, vrsErr := ParseVersion(revID) if vrsErr != nil { return vrsErr } diff --git a/db/change_cache.go b/db/change_cache.go index 0e960e69bf..6bc0a0489c 100644 --- a/db/change_cache.go +++ b/db/change_cache.go @@ -397,7 +397,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { } // First unmarshal the doc (just its metadata, to save time/memory): - syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.userXattrKey(), false) + syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(ctx, docJSON, event.DataType, collection.userXattrKey(), false) if err != nil { // Avoid log noise related to failed unmarshaling of binary documents. if event.DataType != base.MemcachedDataTypeRaw { diff --git a/db/crud.go b/db/crud.go index 1294ce74c1..45221aaa83 100644 --- a/db/crud.go +++ b/db/crud.go @@ -345,7 +345,7 @@ func (db *DatabaseCollectionWithUser) documentRevisionForRequest(ctx context.Con if revID != nil { requestedVersion = *revID } else if cv != nil { - requestedVersion = cv.BlipString() + requestedVersion = cv.String() } if revision.BodyBytes == nil { @@ -3059,7 +3059,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedRev(ctx context.Context, doci // CheckProposedVersion - given DocID and a version in string form, check whether it can be added without conflict. func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, docid, proposedVersionStr string, previousVersionStr string) (status ProposedRevStatus, currentVersion string) { - proposedVersion, err := ParseBlipVersion(proposedVersionStr) + proposedVersion, err := ParseVersion(proposedVersionStr) if err != nil { base.WarnfCtx(ctx, "Couldn't parse proposed version for doc %q / %q: %v", base.UD(docid), proposedVersionStr, err) return ProposedRev_Error, "" @@ -3068,7 +3068,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, var previousVersion Version if previousVersionStr != "" { var err error - previousVersion, err = ParseBlipVersion(previousVersionStr) + previousVersion, err = ParseVersion(previousVersionStr) if err != nil { base.WarnfCtx(ctx, "Couldn't parse previous version for doc %q / %q: %v", base.UD(docid), previousVersionStr, err) return ProposedRev_Error, "" @@ -3100,7 +3100,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, // Conflict, return the current cv. This may be a false positive conflict if the client has replicated // the server cv via a different peer. Client is responsible for performing this check based on the // returned localDocCV - return ProposedRev_Conflict, localDocCV.BlipString() + return ProposedRev_Conflict, localDocCV.String() } } diff --git a/db/database_test.go b/db/database_test.go index a9002dfa1c..bed5a7ec58 100644 --- a/db/database_test.go +++ b/db/database_test.go @@ -391,7 +391,7 @@ func TestCheckProposedVersion(t *testing.T) { newVersion: Version{"other", incrementCas(cvValue, -100)}, previousVersion: &Version{cvSource, incrementCas(cvValue, -50)}, expectedStatus: ProposedRev_Conflict, - expectedRev: Version{cvSource, cvValue}.BlipString(), + expectedRev: Version{cvSource, cvValue}.String(), }, { // conflict - previous version is older than CV @@ -399,7 +399,7 @@ func TestCheckProposedVersion(t *testing.T) { newVersion: Version{"other", incrementCas(cvValue, 100)}, previousVersion: &Version{"other", incrementCas(cvValue, -50)}, expectedStatus: ProposedRev_Conflict, - expectedRev: Version{cvSource, cvValue}.BlipString(), + expectedRev: Version{cvSource, cvValue}.String(), }, } @@ -407,9 +407,9 @@ func TestCheckProposedVersion(t *testing.T) { t.Run(tc.name, func(t *testing.T) { previousVersionStr := "" if tc.previousVersion != nil { - previousVersionStr = tc.previousVersion.BlipString() + previousVersionStr = tc.previousVersion.String() } - status, rev := collection.CheckProposedVersion(ctx, "doc1", tc.newVersion.BlipString(), previousVersionStr) + status, rev := collection.CheckProposedVersion(ctx, "doc1", tc.newVersion.String(), previousVersionStr) assert.Equal(t, tc.expectedStatus, status) assert.Equal(t, tc.expectedRev, rev) }) diff --git a/db/document.go b/db/document.go index f151fbae30..245e0db6f9 100644 --- a/db/document.go +++ b/db/document.go @@ -463,7 +463,7 @@ func UnmarshalDocumentSyncData(data []byte, needHistory bool) (*SyncData, error) // Returns the raw body, in case it's needed for import. // TODO: Using a pool of unmarshal workers may help prevent memory spikes under load -func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey string, needHistory bool) (result *SyncData, rawBody []byte, rawXattrs map[string][]byte, err error) { +func UnmarshalDocumentSyncDataFromFeed(ctx context.Context, data []byte, dataType uint8, userXattrKey string, needHistory bool) (result *SyncData, rawBody []byte, rawXattrs map[string][]byte, err error) { var body []byte // If xattr datatype flag is set, data includes both xattrs and document body. Check for presence of sync xattr. @@ -484,7 +484,7 @@ func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey syncXattr, ok := xattrValues[base.SyncXattrName] if vvXattr, ok := xattrValues[base.VvXattrName]; ok { - docHLV, err := ParseHLVFields(vvXattr) + docHLV, err := ParseHLVFields(ctx, vvXattr) if err != nil { return nil, nil, nil, fmt.Errorf("error unmarshalling HLV: %w", err) } @@ -1119,7 +1119,7 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - docHLV, err := ParseHLVFields(hlvXattrData) + docHLV, err := ParseHLVFields(ctx, hlvXattrData) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) } @@ -1161,7 +1161,7 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - docHLV, err := ParseHLVFields(hlvXattrData) + docHLV, err := ParseHLVFields(ctx, hlvXattrData) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), err)) } @@ -1318,7 +1318,7 @@ func ConstructXattrFromHlv(hlv *HybridLogicalVector) ([]byte, error) { // ParseHLVFields will parse raw bytes of the hlv xattr from persisted version to in memory version, converting the // deltas in pv and mv back into in memory format -func ParseHLVFields(xattr []byte) (*HybridLogicalVector, error) { +func ParseHLVFields(ctx context.Context, xattr []byte) (*HybridLogicalVector, error) { var persistedHLV PersistedHLV err := base.JSONUnmarshal(xattr, &persistedHLV) if err != nil { @@ -1327,11 +1327,11 @@ func ParseHLVFields(xattr []byte) (*HybridLogicalVector, error) { pvMap, deltaErr := PersistedDeltasToMap(persistedHLV.PreviousVersions) if deltaErr != nil { - return nil, fmt.Errorf("error converting pv to map: %v", deltaErr) + base.WarnfCtx(ctx, "Failed to convert persisted version deltas to full version values for previous versions. Falling back to constructing HLV without previous versions PV: %v, Error: %v", persistedHLV.PreviousVersions, deltaErr) } mvMap, deltaErr := PersistedDeltasToMap(persistedHLV.MergeVersions) if deltaErr != nil { - return nil, fmt.Errorf("error converting mv to map: %v", deltaErr) + base.WarnfCtx(ctx, "Failed to convert persisted version deltas to full version values for merge versions. Falling back to constructing HLV without previous versions MV: %v, Error: %v", persistedHLV.MergeVersions, deltaErr) } newHLV := NewHybridLogicalVector() diff --git a/db/document_test.go b/db/document_test.go index 2659fa588f..a67dce41ad 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -240,8 +240,8 @@ const doc_meta_no_vv = `{ }` const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", - "mv":["s_LhRPsa7CpjEvP5zeXTXEBA@0xc0ff05d7ac059a16","s_NqiIe0LekFPLeX4JvTO6Iw@0x1c008cd6"], - "pv":["s_YZvBpEaztom9z5V/hDoeIw@0xf0ff44d6ac059a16"] + "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","0x1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] }` func TestParseVersionVectorSyncData(t *testing.T) { @@ -287,6 +287,33 @@ func TestParseVersionVectorSyncData(t *testing.T) { assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) } +const doc_meta_vv_corrupt = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", + "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] +}` + +func TestParseVersionVectorCorruptDelta(t *testing.T) { + pv := make(map[string]string) + pv["s_YZvBpEaztom9z5V/hDoeIw"] = "0xf0ff44d6ac059a16" + + ctx := base.TestCtx(t) + + sync_meta := []byte(doc_meta_no_vv) + vv_meta := []byte(doc_meta_vv_corrupt) + doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, 1, DocUnmarshalAll) + require.NoError(t, err) + + strCAS := string(base.Uint64CASToLittleEndianHex(123456)) + // assert on doc version vector values + assert.Equal(t, strCAS, doc.SyncData.HLV.CurrentVersionCAS) + assert.Equal(t, strCAS, doc.SyncData.HLV.Version) + assert.Equal(t, "cb06dc003846116d9b66d2ab23887a96", doc.SyncData.HLV.SourceID) + // Passing in corrupt merge versions so this should be nil + assert.Nil(t, doc.SyncData.HLV.MergeVersions) + assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) + +} + // TestRevAndVersion tests marshalling and unmarshalling rev and current version func TestRevAndVersion(t *testing.T) { @@ -478,7 +505,7 @@ func TestDCPDecodeValue(t *testing.T) { require.Nil(t, xattrs) } // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(test.body, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(nil, test.body, base.MemcachedDataTypeXattr, "", false) require.ErrorIs(t, err, test.expectedErr) if test.expectedSyncXattr != nil { require.NotNil(t, result) @@ -503,7 +530,7 @@ func TestInvalidXattrStreamEmptyBody(t *testing.T) { require.Empty(t, xattrs) // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(inputStream, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(nil, inputStream, base.MemcachedDataTypeXattr, "", false) require.Error(t, err) // unexpected end of JSON input require.Nil(t, result) require.Equal(t, emptyBody, rawBody) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 8cd81d99ef..2726dea251 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -81,7 +81,7 @@ func VersionsToDeltas(m map[string]string) []string { val := delta.Value encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(val) vrs := Version{SourceID: key, Value: encodedVal} - vrsList = append(vrsList, vrs.DeltaString()) + vrsList = append(vrsList, vrs.String()) } return vrsList @@ -96,7 +96,7 @@ func PersistedDeltasToMap(vvList []string) (map[string]string, error) { var lastEntryVersion uint64 for _, v := range vvList { - vrs, err := ParseDeltasVersion(v) + vrs, err := ParseVersion(v) if err != nil { return nil, err } @@ -119,8 +119,8 @@ func CreateVersion(source string, version uint64) Version { } } -// ParseBlipVersion will parse source version pair from blip format -func ParseBlipVersion(versionString string) (version Version, err error) { +// ParseVersion will parse source version pair from blip format +func ParseVersion(versionString string) (version Version, err error) { timestampString, sourceBase64, found := strings.Cut(versionString, "@") if !found { return version, fmt.Errorf("Malformed version string %s, delimiter not found", versionString) @@ -143,11 +143,6 @@ func (v Version) String() string { return strconv.FormatUint(v.Value, 16) + "@" + v.SourceID } -// DeltaString returns a pv or mv delta format string representation of version -func (v Version) DeltaString() string { - return v.SourceID + "@" + v.Value -} - // ExtractCurrentVersionFromHLV will take the current version form the HLV struct and return it in the Version struct func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { src, vrs := hlv.GetCurrentVersion() @@ -207,7 +202,7 @@ func (hlv *HybridLogicalVector) GetCurrentVersionString() string { SourceID: hlv.SourceID, Value: hlv.Version, } - return version.BlipString() + return version.String() } // IsVersionKnown checks to see whether the HLV already contains a Version for the provided @@ -390,7 +385,7 @@ func (hlv *HybridLogicalVector) ToHistoryForHLV() string { itemNo := 1 for key, value := range hlv.MergeVersions { vrs := Version{SourceID: key, Value: value} - s.WriteString(vrs.BlipString()) + s.WriteString(vrs.String()) if itemNo < len(hlv.MergeVersions) { s.WriteString(",") } @@ -406,7 +401,7 @@ func (hlv *HybridLogicalVector) ToHistoryForHLV() string { } for key, value := range hlv.PreviousVersions { vrs := Version{SourceID: key, Value: value} - s.WriteString(vrs.BlipString()) + s.WriteString(vrs.String()) if itemNo < len(hlv.PreviousVersions) { s.WriteString(",") } @@ -511,7 +506,7 @@ func parseVectorValues(vectorStr string) (versions []Version, err error) { if len(v) > 0 && v[0] == ' ' { v = v[1:] } - version, err := ParseBlipVersion(v) + version, err := ParseVersion(v) if err != nil { return nil, err } diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index bfed01d474..6522da2f1b 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -513,7 +513,7 @@ func TestVersionDeltaCalculation(t *testing.T) { require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err := ParseHLVFields(vvXattr) + memHLV, err := ParseHLVFields(nil, vvXattr) require.NoError(t, err) assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) @@ -538,7 +538,7 @@ func TestVersionDeltaCalculation(t *testing.T) { vvXattr, err = ConstructXattrFromHlv(&hlv2) require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err = ParseHLVFields(vvXattr) + memHLV, err = ParseHLVFields(nil, vvXattr) require.NoError(t, err) // assert in memory hlv is as expected assert.Equal(t, expSrc, memHLV.SourceID) diff --git a/db/import_listener.go b/db/import_listener.go index a92c7b72ec..7104c5e2f5 100644 --- a/db/import_listener.go +++ b/db/import_listener.go @@ -168,7 +168,7 @@ func (il *importListener) ProcessFeedEvent(event sgbucket.FeedEvent) (shouldPers } func (il *importListener) ImportFeedEvent(ctx context.Context, collection *DatabaseCollectionWithUser, event sgbucket.FeedEvent) { - syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(event.Value, event.DataType, collection.userXattrKey(), false) + syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(ctx, event.Value, event.DataType, collection.userXattrKey(), false) if err != nil { if errors.Is(err, sgbucket.ErrEmptyMetadata) { base.WarnfCtx(ctx, "Unexpected empty metadata when processing feed event. docid: %s opcode: %v datatype:%v", base.UD(event.Key), event.Opcode, event.DataType) diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go index 4c6e9f9a2e..1edd02ace4 100644 --- a/db/utilities_hlv_testing.go +++ b/db/utilities_hlv_testing.go @@ -150,7 +150,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // pv for _, versionStr := range strings.Split(pvString, ",") { - version, err := ParseBlipVersion(versionStr) + version, err := ParseVersion(versionStr) require.NoError(t, err) pv[EncodeSource(version.SourceID)] = version.Value } @@ -158,7 +158,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // mv if mvString != "" { for _, versionStr := range strings.Split(mvString, ",") { - version, err := ParseBlipVersion(versionStr) + version, err := ParseVersion(versionStr) require.NoError(t, err) mv[EncodeSource(version.SourceID)] = version.Value } @@ -168,7 +168,7 @@ func ParseTestHistory(t *testing.T, historyString string) (pv HLVVersions, mv HL // Requires that the CV for the provided HLV matches the expected CV (sent in simplified test format) func RequireCVEqual(t *testing.T, hlv *HybridLogicalVector, expectedCV string) { - testVersion, err := ParseBlipVersion(expectedCV) + testVersion, err := ParseVersion(expectedCV) require.NoError(t, err) require.Equal(t, EncodeSource(testVersion.SourceID), hlv.SourceID) require.Equal(t, testVersion.Value, hlv.Version) diff --git a/rest/importuserxattrtest/import_test.go b/rest/importuserxattrtest/import_test.go index 6e40678f0a..fc7f53171e 100644 --- a/rest/importuserxattrtest/import_test.go +++ b/rest/importuserxattrtest/import_test.go @@ -412,7 +412,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value := sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) require.NoError(t, err) assert.Equal(t, syncXattr, string(rawXattrs[base.SyncXattrName])) assert.Equal(t, uint64(200), syncData.Sequence) @@ -425,7 +425,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) @@ -436,7 +436,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { xattrs = []sgbucket.Xattr{} value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 625f5c2496..108ee0d2fa 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -2468,7 +2468,7 @@ func (v DocVersion) GetRev(useHLV bool) string { if v.CV.SourceID == "" { return "" } - return v.CV.BlipString() + return v.CV.String() } else { return v.RevTreeID } diff --git a/rest/utilities_testing_blip_client.go b/rest/utilities_testing_blip_client.go index c5047ef05d..d433d96ebc 100644 --- a/rest/utilities_testing_blip_client.go +++ b/rest/utilities_testing_blip_client.go @@ -175,7 +175,7 @@ func (btcr *BlipTesterCollectionClient) NewBlipTesterDoc(revID string, body []by } func VersionFromRevID(revID string) db.Version { - version, err := db.ParseBlipVersion(revID) + version, err := db.ParseVersion(revID) if err != nil { panic(err) } @@ -924,7 +924,7 @@ func (btc *BlipTesterCollectionClient) PushRev(docID string, parentVersion DocVe func (btc *BlipTesterCollectionClient) requireRevID(expected DocVersion, revID string) { if btc.UseHLV() { - require.Equal(btc.parent.rt.TB(), expected.CV.BlipString(), revID) + require.Equal(btc.parent.rt.TB(), expected.CV.String(), revID) } else { require.Equal(btc.parent.rt.TB(), expected.RevTreeID, revID) } @@ -1193,7 +1193,7 @@ func (btc *BlipTesterClient) AssertOnBlipHistory(t *testing.T, msg *blip.Message require.NoError(t, err) if subProtocol >= db.CBMobileReplicationV4 { // history could be empty a lot of the time in HLV messages as updates from the same source won't populate previous versions if msg.Properties[db.RevMessageHistory] != "" { - assert.Equal(t, docVersion.CV.BlipString(), msg.Properties[db.RevMessageHistory]) + assert.Equal(t, docVersion.CV.String(), msg.Properties[db.RevMessageHistory]) } } else { assert.Equal(t, docVersion.RevTreeID, msg.Properties[db.RevMessageHistory]) @@ -1207,7 +1207,7 @@ func (btc *BlipTesterCollectionClient) GetVersion(docID string, docVersion DocVe if doc, ok := btc.docs[docID]; ok { if doc.revMode == revModeHLV { - if doc.getCurrentRevID() == docVersion.CV.BlipString() { + if doc.getCurrentRevID() == docVersion.CV.String() { return doc.body, true } } else { @@ -1306,7 +1306,7 @@ func (btr *BlipTesterReplicator) storeMessage(msg *blip.Message) { func (btc *BlipTesterCollectionClient) WaitForBlipRevMessage(docID string, version DocVersion) (msg *blip.Message) { var revID string if btc.UseHLV() { - revID = version.CV.BlipString() + revID = version.CV.String() } else { revID = version.RevTreeID } From 3d1050294694bebe99c1c98a333ef139b5df31d4 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 12 Sep 2024 11:30:59 +0100 Subject: [PATCH 07/19] clean up --- db/document_test.go | 4 ++-- db/hybrid_logical_vector.go | 2 +- db/hybrid_logical_vector_test.go | 5 +++-- rest/importuserxattrtest/import_test.go | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/db/document_test.go b/db/document_test.go index a67dce41ad..bc0ea0456e 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -505,7 +505,7 @@ func TestDCPDecodeValue(t *testing.T) { require.Nil(t, xattrs) } // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(nil, test.body, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), test.body, base.MemcachedDataTypeXattr, "", false) require.ErrorIs(t, err, test.expectedErr) if test.expectedSyncXattr != nil { require.NotNil(t, result) @@ -530,7 +530,7 @@ func TestInvalidXattrStreamEmptyBody(t *testing.T) { require.Empty(t, xattrs) // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(nil, inputStream, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), inputStream, base.MemcachedDataTypeXattr, "", false) require.Error(t, err) // unexpected end of JSON input require.Nil(t, result) require.Equal(t, emptyBody, rawBody) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 2726dea251..8864dfdc7e 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -119,7 +119,7 @@ func CreateVersion(source string, version uint64) Version { } } -// ParseVersion will parse source version pair from blip format +// ParseVersion will parse source version pair from string format func ParseVersion(versionString string) (version Version, err error) { timestampString, sourceBase64, found := strings.Cut(versionString, "@") if !found { diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index 6522da2f1b..8e0bd9f83a 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -471,6 +471,7 @@ func TestParseCBLVersion(t *testing.T) { // the corresponding element in the original pvMap // - Do the same as above but for nil maps func TestVersionDeltaCalculation(t *testing.T) { + ctx := base.TestCtx(t) src1 := "src1" src2 := "src2" src3 := "src3" @@ -513,7 +514,7 @@ func TestVersionDeltaCalculation(t *testing.T) { require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err := ParseHLVFields(nil, vvXattr) + memHLV, err := ParseHLVFields(ctx, vvXattr) require.NoError(t, err) assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) @@ -538,7 +539,7 @@ func TestVersionDeltaCalculation(t *testing.T) { vvXattr, err = ConstructXattrFromHlv(&hlv2) require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err = ParseHLVFields(nil, vvXattr) + memHLV, err = ParseHLVFields(ctx, vvXattr) require.NoError(t, err) // assert in memory hlv is as expected assert.Equal(t, expSrc, memHLV.SourceID) diff --git a/rest/importuserxattrtest/import_test.go b/rest/importuserxattrtest/import_test.go index fc7f53171e..6b419d0809 100644 --- a/rest/importuserxattrtest/import_test.go +++ b/rest/importuserxattrtest/import_test.go @@ -412,7 +412,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value := sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) require.NoError(t, err) assert.Equal(t, syncXattr, string(rawXattrs[base.SyncXattrName])) assert.Equal(t, uint64(200), syncData.Sequence) @@ -425,7 +425,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) @@ -436,7 +436,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { xattrs = []sgbucket.Xattr{} value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(nil, value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) From ac139eb9d0565bf58901a87ed4ee7ffdd7b376ba Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 12 Sep 2024 11:44:25 +0100 Subject: [PATCH 08/19] fix from rebase --- db/document_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/document_test.go b/db/document_test.go index bc0ea0456e..cc5dbc2a0b 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -300,7 +300,7 @@ func TestParseVersionVectorCorruptDelta(t *testing.T) { sync_meta := []byte(doc_meta_no_vv) vv_meta := []byte(doc_meta_vv_corrupt) - doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, 1, DocUnmarshalAll) + doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, 1, DocUnmarshalAll) require.NoError(t, err) strCAS := string(base.Uint64CASToLittleEndianHex(123456)) From 7af5e2b4227d627f9043263b71408714dd23cb79 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 13 Sep 2024 11:51:19 +0100 Subject: [PATCH 09/19] safety for decoding delta value --- base/util.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/base/util.go b/base/util.go index 89fcb7f572..56580f15bf 100644 --- a/base/util.go +++ b/base/util.go @@ -1020,14 +1020,15 @@ func HexCasToUint64(cas string) uint64 { // HexCasToUint64ForDelta will convert hex cas to uint64 accounting for any stripped zeros in delta calculation func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { + var decoded []byte if len(casByte) <= 2 { - return 0, fmt.Errorf("hex value is too short.") + return 0, fmt.Errorf("hex value is too short") } if casByte[0] != '0' || casByte[1] != 'x' { return 0, fmt.Errorf("incorrect hex value, leading 0x is expected") } - // as we strip any zeros iff the end of the hex value for deltas, the input delta could be odd length + // as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length if len(casByte)%2 != 0 { evenHexLen := make([]byte, len(casByte), len(casByte)+1) copy(evenHexLen, casByte) @@ -1035,9 +1036,21 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { casByte = evenHexLen } - decoded := make([]byte, 8) - _, err := hex.Decode(decoded, casByte[2:]) - if err != nil { + // create byte array for decoding into + decodedLen := hex.DecodedLen(len(casByte[2:])) + // binary.LittleEndian.Uint64 expects length 8 byte array, if larger we should error, if smaller + // (because of stripped 0's then we should make it length 8). + if decodedLen > 8 { + return 0, fmt.Errorf("corrupt hex value, decoded length larger than expected") + } + if decodedLen < 8 { + // can be less than 8 given we have stripped the 0's for some values, in this case we need to ensure large eniough + decoded = make([]byte, 8) + } else { + decoded = make([]byte, decodedLen) + } + + if _, err := hex.Decode(decoded, casByte[2:]); err != nil { return 0, err } res := binary.LittleEndian.Uint64(decoded) From 857ab828825a80bea80f3b17068f51182599d492 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 13 Sep 2024 12:09:07 +0100 Subject: [PATCH 10/19] lint error Signed-off-by: Gregory Newman-Smith --- db/document_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/document_test.go b/db/document_test.go index cc5dbc2a0b..29b8a219f5 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -300,7 +300,7 @@ func TestParseVersionVectorCorruptDelta(t *testing.T) { sync_meta := []byte(doc_meta_no_vv) vv_meta := []byte(doc_meta_vv_corrupt) - doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, 1, DocUnmarshalAll) + doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll) require.NoError(t, err) strCAS := string(base.Uint64CASToLittleEndianHex(123456)) From 665b76d20435a154fb60be826c8bdd78b1d22e6c Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Tue, 17 Sep 2024 17:09:32 +0100 Subject: [PATCH 11/19] updates to remove leading 0x in deltas --- base/util.go | 17 +++++++++++------ db/document_test.go | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/base/util.go b/base/util.go index 56580f15bf..94896e0551 100644 --- a/base/util.go +++ b/base/util.go @@ -1024,9 +1024,6 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { if len(casByte) <= 2 { return 0, fmt.Errorf("hex value is too short") } - if casByte[0] != '0' || casByte[1] != 'x' { - return 0, fmt.Errorf("incorrect hex value, leading 0x is expected") - } // as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length if len(casByte)%2 != 0 { @@ -1037,7 +1034,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { } // create byte array for decoding into - decodedLen := hex.DecodedLen(len(casByte[2:])) + decodedLen := hex.DecodedLen(len(casByte)) // binary.LittleEndian.Uint64 expects length 8 byte array, if larger we should error, if smaller // (because of stripped 0's then we should make it length 8). if decodedLen > 8 { @@ -1050,7 +1047,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { decoded = make([]byte, decodedLen) } - if _, err := hex.Decode(decoded, casByte[2:]); err != nil { + if _, err := hex.Decode(decoded, casByte); err != nil { return 0, err } res := binary.LittleEndian.Uint64(decoded) @@ -1059,7 +1056,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { // Uint64ToLittleEndianHexAndStripZeros will convert a uint64 type to little endian hex, stripping any zeros off the end func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string { - hexCas := Uint64CASToLittleEndianHex(cas) + hexCas := Uint64CASToLittleEndianHexNo0x(cas) i := len(hexCas) - 1 for i > 2 && hexCas[i] == '0' { @@ -1078,6 +1075,14 @@ func HexToBase64(s string) ([]byte, error) { return encoded, nil } +func Uint64CASToLittleEndianHexNo0x(cas uint64) []byte { + littleEndian := make([]byte, 8) + binary.LittleEndian.PutUint64(littleEndian, cas) + encodedArray := make([]byte, hex.EncodedLen(8)) + _ = hex.Encode(encodedArray, littleEndian) + return encodedArray +} + func CasToString(cas uint64) string { return string(Uint64CASToLittleEndianHex(cas)) } diff --git a/db/document_test.go b/db/document_test.go index 29b8a219f5..a7646d50e2 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -240,8 +240,8 @@ const doc_meta_no_vv = `{ }` const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", - "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","0x1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], - "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] + "mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] }` func TestParseVersionVectorSyncData(t *testing.T) { @@ -288,8 +288,8 @@ func TestParseVersionVectorSyncData(t *testing.T) { } const doc_meta_vv_corrupt = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2ab23887a96","ver":"0x40e2010000000000", - "mv":["0xc0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], - "pv":["0xf0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] + "mv":["c0ff05d7ac059a16@s_LhRPsa7CpjEvP5zeXTXEBA","1c008cd61c008cd61c008cd6@s_NqiIe0LekFPLeX4JvTO6Iw"], + "pv":["f0ff44d6ac059a16@s_YZvBpEaztom9z5V/hDoeIw"] }` func TestParseVersionVectorCorruptDelta(t *testing.T) { From da52e6a8cbd21a15baceecbb29ad65642db29d69 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Wed, 18 Sep 2024 15:03:35 +0100 Subject: [PATCH 12/19] fix comment --- db/hybrid_logical_vector.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 8864dfdc7e..63524f4ed3 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -150,8 +150,7 @@ func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { return &currVersion } -// PersistedHybridLogicalVector is the marshalled format of HybridLogicalVector. -// This representation needs to be kept in sync with XDCR. +// HybridLogicalVector is the in memory format for the hLv. type HybridLogicalVector struct { CurrentVersionCAS uint64 // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication ImportCAS uint64 // Set when an import modifies the document CAS but preserves the HLV (import of a version replicated by XDCR) From 547c706b1a9c87107a0107c6cb05552276258f39 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 20 Sep 2024 11:04:04 +0100 Subject: [PATCH 13/19] custom marahal/unmarahal --- db/crud_test.go | 12 +++---- db/document.go | 19 ++++++---- db/document_test.go | 15 ++------ db/hybrid_logical_vector.go | 59 ++++++++++++++++++++++++++++++++ db/hybrid_logical_vector_test.go | 2 +- 5 files changed, 81 insertions(+), 26 deletions(-) diff --git a/db/crud_test.go b/db/crud_test.go index f4cc5fa9cb..1f99000b88 100644 --- a/db/crud_test.go +++ b/db/crud_test.go @@ -1822,9 +1822,9 @@ func TestPutExistingCurrentVersion(t *testing.T) { // create a version larger than the allocated version above incomingVersion := docUpdateVersionInt + 10 incomingHLV := HybridLogicalVector{ - SourceID: "test", - Version: incomingVersion, - PreviousVersions: pv, + SourceID: "test", + Version: incomingVersion, + PrevVersions: PrevVersions{PreviousVersions: pv}, } doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil) @@ -1940,9 +1940,9 @@ func TestPutExistingCurrentVersionWithNoExistingDoc(t *testing.T) { // create a version larger than the allocated version above incomingVersion := uint64(2 + 10) incomingHLV := HybridLogicalVector{ - SourceID: "test", - Version: incomingVersion, - PreviousVersions: pv, + SourceID: "test", + Version: incomingVersion, + PrevVersions: PrevVersions{PreviousVersions: pv}, } // call PutExistingCurrentVersion with empty existing doc doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, &sgbucket.BucketDocument{}) diff --git a/db/document.go b/db/document.go index 245e0db6f9..35d9d5e7ff 100644 --- a/db/document.go +++ b/db/document.go @@ -1119,11 +1119,12 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - docHLV, err := ParseHLVFields(ctx, hlvXattrData) + //docHLV, err := ParseHLVFields(ctx, hlvXattrData) + err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) } - doc.HLV = docHLV + //doc.HLV = docHLV } if virtualXattr != nil { var revSeqNo string @@ -1161,11 +1162,12 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - docHLV, err := ParseHLVFields(ctx, hlvXattrData) + //docHLV, err := ParseHLVFields(ctx, hlvXattrData) + err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), err)) } - doc.HLV = docHLV + //doc.HLV = docHLV } if len(globalSyncData) > 0 { if err := base.JSONUnmarshal(globalSyncData, &doc.GlobalSyncData); err != nil { @@ -1256,11 +1258,16 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl } } if doc.SyncData.HLV != nil { - // convert in memory hlv into persisted version, converting maps to deltas - vvXattr, err = ConstructXattrFromHlv(doc.HLV) + vvXattr, err = base.JSONMarshal(doc.SyncData.HLV) + //fmt.Println(syncXattr) if err != nil { return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) } + // convert in memory hlv into persisted version, converting maps to deltas + //vvXattr, err = ConstructXattrFromHlv(doc.HLV) + //if err != nil { + // return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) + //} } // assign any attachments we have stored in document sync data to global sync data // then nil the sync data attachments to prevent marshalling of it diff --git a/db/document_test.go b/db/document_test.go index a7646d50e2..69330408ae 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -293,24 +293,13 @@ const doc_meta_vv_corrupt = `{"cvCas":"0x40e2010000000000","src":"cb06dc00384611 }` func TestParseVersionVectorCorruptDelta(t *testing.T) { - pv := make(map[string]string) - pv["s_YZvBpEaztom9z5V/hDoeIw"] = "0xf0ff44d6ac059a16" ctx := base.TestCtx(t) sync_meta := []byte(doc_meta_no_vv) vv_meta := []byte(doc_meta_vv_corrupt) - doc, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll) - require.NoError(t, err) - - strCAS := string(base.Uint64CASToLittleEndianHex(123456)) - // assert on doc version vector values - assert.Equal(t, strCAS, doc.SyncData.HLV.CurrentVersionCAS) - assert.Equal(t, strCAS, doc.SyncData.HLV.Version) - assert.Equal(t, "cb06dc003846116d9b66d2ab23887a96", doc.SyncData.HLV.SourceID) - // Passing in corrupt merge versions so this should be nil - assert.Nil(t, doc.SyncData.HLV.MergeVersions) - assert.True(t, reflect.DeepEqual(pv, doc.SyncData.HLV.PreviousVersions)) + _, err := unmarshalDocumentWithXattrs(ctx, "doc1", nil, sync_meta, vv_meta, nil, nil, nil, nil, 1, DocUnmarshalAll) + require.Error(t, err) } diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 63524f4ed3..180e7b63dd 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -150,6 +150,65 @@ func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { return &currVersion } +type PrevVersions struct { + PreviousVersions HLVVersions `json:"pv,omitempty"` +} + +type MVersions struct { + MergeVersions HLVVersions `json:"mv,omitempty"` +} + +type HybridLogicalVectorJSON struct { + *HLVAlias +} + +func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { + var hlvJSON HybridLogicalVectorJSON + var alias HLVAlias + alias = (HLVAlias)(hlv) + hlvJSON.HLVAlias = &alias + return base.JSONMarshal(hlvJSON) +} + +func (ver HLVVersions) MarshalJSON() ([]byte, error) { + fmt.Println("inside") + var verList []string + if len(ver) > 0 { + verList = make([]string, len(ver)) + verList = VersionsToDeltas(ver) + return base.JSONMarshal(verList) + } + return nil, nil +} + +func (ver *HLVVersions) UnmarshalJSON(inputjson []byte) error { + + var verList []string + err := base.JSONUnmarshal(inputjson, &verList) + if err != nil { + return err + } + verMap, deltaErr := PersistedDeltasToMap(verList) + if deltaErr != nil { + return deltaErr // need typed erro to assert on + } + *ver = verMap + return nil +} + +func (hlv *HybridLogicalVector) UnmarshalJSON(data []byte) error { + + var hlvJSON *HybridLogicalVectorJSON + err := base.JSONUnmarshal(data, &hlvJSON) + if err != nil { + return err + } + if hlvJSON.HLVAlias != nil { + *hlv = HybridLogicalVector(*hlvJSON.HLVAlias) + } + return nil +} + // HybridLogicalVector is the in memory format for the hLv. type HybridLogicalVector struct { CurrentVersionCAS uint64 // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index 8e0bd9f83a..fb28559e24 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -487,7 +487,7 @@ func TestVersionDeltaCalculation(t *testing.T) { v5 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) // make map of source to version - pvMap := make(map[string]string) + pvMap := make(HLVVersions) pvMap[src1] = v1 pvMap[src2] = v2 pvMap[src3] = v3 From b94c4c8b276019f70ff06fda43d31b23cfdfa9e0 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 20 Sep 2024 13:05:53 +0100 Subject: [PATCH 14/19] lint fix --- db/hybrid_logical_vector.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 180e7b63dd..0bb87715e4 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -171,10 +171,8 @@ func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { } func (ver HLVVersions) MarshalJSON() ([]byte, error) { - fmt.Println("inside") - var verList []string + var verList = make([]string, len(ver)) if len(ver) > 0 { - verList = make([]string, len(ver)) verList = VersionsToDeltas(ver) return base.JSONMarshal(verList) } From 902462f397484c6a820da934be670caf9be85996 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 20 Sep 2024 13:10:31 +0100 Subject: [PATCH 15/19] lint --- db/hybrid_logical_vector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 0bb87715e4..9f4e8f0a44 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -171,7 +171,7 @@ func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { } func (ver HLVVersions) MarshalJSON() ([]byte, error) { - var verList = make([]string, len(ver)) + var verList []string if len(ver) > 0 { verList = VersionsToDeltas(ver) return base.JSONMarshal(verList) From addb7673082a5f28222fd86d296c2fda917e4b0e Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 4 Oct 2024 11:43:06 +0100 Subject: [PATCH 16/19] updates to marshal and unmarshal functions --- db/crud_test.go | 12 +- db/document.go | 66 +-------- db/document_test.go | 4 +- db/hybrid_logical_vector.go | 228 ++++++++++--------------------- db/hybrid_logical_vector_test.go | 24 ++-- db/utilities_hlv_testing.go | 6 +- rest/api_test.go | 14 +- 7 files changed, 103 insertions(+), 251 deletions(-) diff --git a/db/crud_test.go b/db/crud_test.go index 1f99000b88..f4cc5fa9cb 100644 --- a/db/crud_test.go +++ b/db/crud_test.go @@ -1822,9 +1822,9 @@ func TestPutExistingCurrentVersion(t *testing.T) { // create a version larger than the allocated version above incomingVersion := docUpdateVersionInt + 10 incomingHLV := HybridLogicalVector{ - SourceID: "test", - Version: incomingVersion, - PrevVersions: PrevVersions{PreviousVersions: pv}, + SourceID: "test", + Version: incomingVersion, + PreviousVersions: pv, } doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, nil) @@ -1940,9 +1940,9 @@ func TestPutExistingCurrentVersionWithNoExistingDoc(t *testing.T) { // create a version larger than the allocated version above incomingVersion := uint64(2 + 10) incomingHLV := HybridLogicalVector{ - SourceID: "test", - Version: incomingVersion, - PrevVersions: PrevVersions{PreviousVersions: pv}, + SourceID: "test", + Version: incomingVersion, + PreviousVersions: pv, } // call PutExistingCurrentVersion with empty existing doc doc, cv, _, err := collection.PutExistingCurrentVersion(ctx, newDoc, incomingHLV, &sgbucket.BucketDocument{}) diff --git a/db/document.go b/db/document.go index 35d9d5e7ff..385f7e0f6c 100644 --- a/db/document.go +++ b/db/document.go @@ -484,11 +484,10 @@ func UnmarshalDocumentSyncDataFromFeed(ctx context.Context, data []byte, dataTyp syncXattr, ok := xattrValues[base.SyncXattrName] if vvXattr, ok := xattrValues[base.VvXattrName]; ok { - docHLV, err := ParseHLVFields(ctx, vvXattr) + err = base.JSONUnmarshal(vvXattr, &hlv) if err != nil { return nil, nil, nil, fmt.Errorf("error unmarshalling HLV: %w", err) } - hlv = docHLV } if ok && len(syncXattr) > 0 { @@ -1119,12 +1118,10 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - //docHLV, err := ParseHLVFields(ctx, hlvXattrData) err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalAll/Sync). Error: %v", base.UD(doc.ID), err)) } - //doc.HLV = docHLV } if virtualXattr != nil { var revSeqNo string @@ -1162,12 +1159,10 @@ func (doc *Document) UnmarshalWithXattrs(ctx context.Context, data, syncXattrDat } if hlvXattrData != nil { // parse the raw bytes of the hlv and convert deltas back to full values in memory - //docHLV, err := ParseHLVFields(ctx, hlvXattrData) err := base.JSONUnmarshal(hlvXattrData, &doc.HLV) if err != nil { return pkgerrors.WithStack(base.RedactErrorf("Failed to unmarshal HLV during UnmarshalWithXattrs() doc with id: %s (DocUnmarshalNoHistory). Error: %v", base.UD(doc.ID), err)) } - //doc.HLV = docHLV } if len(globalSyncData) > 0 { if err := base.JSONUnmarshal(globalSyncData, &doc.GlobalSyncData); err != nil { @@ -1259,15 +1254,9 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl } if doc.SyncData.HLV != nil { vvXattr, err = base.JSONMarshal(doc.SyncData.HLV) - //fmt.Println(syncXattr) if err != nil { return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) } - // convert in memory hlv into persisted version, converting maps to deltas - //vvXattr, err = ConstructXattrFromHlv(doc.HLV) - //if err != nil { - // return nil, nil, nil, nil, nil, pkgerrors.WithStack(base.RedactErrorf("Failed to MarshalWithXattrs() doc vv with id: %s. Error: %v", base.UD(doc.ID), err)) - //} } // assign any attachments we have stored in document sync data to global sync data // then nil the sync data attachments to prevent marshalling of it @@ -1299,59 +1288,6 @@ func (doc *Document) MarshalWithXattrs() (data, syncXattr, vvXattr, mouXattr, gl return data, syncXattr, vvXattr, mouXattr, globalXattr, nil } -// ConstructXattrFromHlv will build a persisted hlv from the in memory hlv. Converting the pv and mv maps to deltas -func ConstructXattrFromHlv(hlv *HybridLogicalVector) ([]byte, error) { - var persistedHLV PersistedHLV - - persistedHLV.SourceID = hlv.SourceID - persistedHLV.Version = hlv.Version - persistedHLV.ImportCAS = hlv.ImportCAS - persistedHLV.CurrentVersionCAS = hlv.CurrentVersionCAS - if len(hlv.PreviousVersions) > 0 { - persistedHLV.PreviousVersions = make([]string, len(hlv.PreviousVersions)) - persistedHLV.PreviousVersions = VersionsToDeltas(hlv.PreviousVersions) - } - if len(hlv.MergeVersions) > 0 { - persistedHLV.MergeVersions = make([]string, len(hlv.MergeVersions)) - persistedHLV.MergeVersions = VersionsToDeltas(hlv.MergeVersions) - } - - hlvData, err := base.JSONMarshal(&persistedHLV) - if err != nil { - return nil, err - } - return hlvData, nil -} - -// ParseHLVFields will parse raw bytes of the hlv xattr from persisted version to in memory version, converting the -// deltas in pv and mv back into in memory format -func ParseHLVFields(ctx context.Context, xattr []byte) (*HybridLogicalVector, error) { - var persistedHLV PersistedHLV - err := base.JSONUnmarshal(xattr, &persistedHLV) - if err != nil { - return nil, err - } - - pvMap, deltaErr := PersistedDeltasToMap(persistedHLV.PreviousVersions) - if deltaErr != nil { - base.WarnfCtx(ctx, "Failed to convert persisted version deltas to full version values for previous versions. Falling back to constructing HLV without previous versions PV: %v, Error: %v", persistedHLV.PreviousVersions, deltaErr) - } - mvMap, deltaErr := PersistedDeltasToMap(persistedHLV.MergeVersions) - if deltaErr != nil { - base.WarnfCtx(ctx, "Failed to convert persisted version deltas to full version values for merge versions. Falling back to constructing HLV without previous versions MV: %v, Error: %v", persistedHLV.MergeVersions, deltaErr) - } - - newHLV := NewHybridLogicalVector() - newHLV.SourceID = persistedHLV.SourceID - newHLV.Version = persistedHLV.Version - newHLV.CurrentVersionCAS = persistedHLV.CurrentVersionCAS - newHLV.ImportCAS = persistedHLV.ImportCAS - newHLV.PreviousVersions = pvMap - newHLV.MergeVersions = mvMap - - return &newHLV, nil -} - // computeMetadataOnlyUpdate computes a new metadataOnlyUpdate based on the existing document's CAS and metadataOnlyUpdate func computeMetadataOnlyUpdate(currentCas uint64, revNo uint64, currentMou *MetadataOnlyUpdate) *MetadataOnlyUpdate { var prevCas string diff --git a/db/document_test.go b/db/document_test.go index 69330408ae..64c92d91c6 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -247,8 +247,8 @@ const doc_meta_vv = `{"cvCas":"0x40e2010000000000","src":"cb06dc003846116d9b66d2 func TestParseVersionVectorSyncData(t *testing.T) { mv := make(HLVVersions) pv := make(HLVVersions) - mv["s_LhRPsa7CpjEvP5zeXTXEBA"] = 1628620455147864000 //"c0ff05d7ac059a16" - mv["s_NqiIe0LekFPLeX4JvTO6Iw"] = 1628620455139868700 + mv["s_LhRPsa7CpjEvP5zeXTXEBA"] = 1628620455147864000 + mv["s_NqiIe0LekFPLeX4JvTO6Iw"] = 1628620458747363292 pv["s_YZvBpEaztom9z5V/hDoeIw"] = 1628620455135215600 ctx := base.TestCtx(t) diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 9f4e8f0a44..614300adaf 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -32,7 +32,7 @@ type Version struct { } // VersionsDeltas will be sorted by version, first entry will be fill version then after that will be calculated deltas -type VersionsDeltas []DecodedVersion +type VersionsDeltas []Version func (vde VersionsDeltas) Len() int { return len(vde) } @@ -48,14 +48,14 @@ func (vde VersionsDeltas) Less(i, j int) bool { } // VersionDeltas calculate the deltas of input map -func VersionDeltas(versions map[string]string) VersionsDeltas { +func VersionDeltas(versions map[string]uint64) VersionsDeltas { if versions == nil { return nil } vdm := make(VersionsDeltas, 0, len(versions)) for src, vrs := range versions { - vdm = append(vdm, CreateDecodedVersion(src, base.HexCasToUint64(vrs))) + vdm = append(vdm, CreateVersion(src, vrs)) } // sort the list @@ -69,7 +69,7 @@ func VersionDeltas(versions map[string]string) VersionsDeltas { } // VersionsToDeltas will calculate deltas from the input map (pv or mv). Then will return the deltas in persisted format -func VersionsToDeltas(m map[string]string) []string { +func VersionsToDeltas(m map[string]uint64) []string { if len(m) == 0 { return nil } @@ -80,33 +80,32 @@ func VersionsToDeltas(m map[string]string) []string { key := delta.SourceID val := delta.Value encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(val) - vrs := Version{SourceID: key, Value: encodedVal} - vrsList = append(vrsList, vrs.String()) + listItem := encodedVal + "@" + key + vrsList = append(vrsList, listItem) } return vrsList } // PersistedDeltasToMap converts the list of deltas in pv or mv from the bucket back from deltas into full versions in map format -func PersistedDeltasToMap(vvList []string) (map[string]string, error) { - vv := make(map[string]string) +func PersistedDeltasToMap(vvList []string) (map[string]uint64, error) { + vv := make(map[string]uint64) if len(vvList) == 0 { return vv, nil } var lastEntryVersion uint64 for _, v := range vvList { - vrs, err := ParseVersion(v) - if err != nil { - return nil, err + timestampString, sourceBase64, found := strings.Cut(v, "@") + if !found { + return nil, fmt.Errorf("Malformed version string %s, delimiter not found", v) } - ver, err := base.HexCasToUint64ForDelta([]byte(vrs.Value)) + ver, err := base.HexCasToUint64ForDelta([]byte(timestampString)) if err != nil { return nil, err } lastEntryVersion = ver + lastEntryVersion - calcVer := base.CasToString(lastEntryVersion) - vv[vrs.SourceID] = calcVer + vv[sourceBase64] = lastEntryVersion } return vv, nil } @@ -150,63 +149,6 @@ func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { return &currVersion } -type PrevVersions struct { - PreviousVersions HLVVersions `json:"pv,omitempty"` -} - -type MVersions struct { - MergeVersions HLVVersions `json:"mv,omitempty"` -} - -type HybridLogicalVectorJSON struct { - *HLVAlias -} - -func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { - var hlvJSON HybridLogicalVectorJSON - var alias HLVAlias - alias = (HLVAlias)(hlv) - hlvJSON.HLVAlias = &alias - return base.JSONMarshal(hlvJSON) -} - -func (ver HLVVersions) MarshalJSON() ([]byte, error) { - var verList []string - if len(ver) > 0 { - verList = VersionsToDeltas(ver) - return base.JSONMarshal(verList) - } - return nil, nil -} - -func (ver *HLVVersions) UnmarshalJSON(inputjson []byte) error { - - var verList []string - err := base.JSONUnmarshal(inputjson, &verList) - if err != nil { - return err - } - verMap, deltaErr := PersistedDeltasToMap(verList) - if deltaErr != nil { - return deltaErr // need typed erro to assert on - } - *ver = verMap - return nil -} - -func (hlv *HybridLogicalVector) UnmarshalJSON(data []byte) error { - - var hlvJSON *HybridLogicalVectorJSON - err := base.JSONUnmarshal(data, &hlvJSON) - if err != nil { - return err - } - if hlvJSON.HLVAlias != nil { - *hlv = HybridLogicalVector(*hlvJSON.HLVAlias) - } - return nil -} - // HybridLogicalVector is the in memory format for the hLv. type HybridLogicalVector struct { CurrentVersionCAS uint64 // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication @@ -217,24 +159,15 @@ type HybridLogicalVector struct { PreviousVersions HLVVersions // map of previous versions for fast efficient lookup } -type BucketVector struct { - CurrentVersionCAS string `json:"cvCas,omitempty"` - ImportCAS string `json:"importCAS,omitempty"` - SourceID string `json:"src"` - Version string `json:"ver"` - MergeVersions map[string]string `json:"mv,omitempty"` - PreviousVersions map[string]string `json:"pv,omitempty"` -} - -// PersistedHLV is the version of the version vector that is persisted to the bucket, pv and mv will be lists of source version pairs in the bucket -type PersistedHLV struct { - CurrentVersionCAS string `json:"cvCas,omitempty"` // current version cas (or cvCAS) stores the current CAS in little endian hex format at the time of replication - ImportCAS string `json:"importCAS,omitempty"` // Set when an import modifies the document CAS but preserves the HLV (import of a version replicated by XDCR) - SourceID string `json:"src"` // source bucket uuid in (base64 encoded format) of where this entry originated from - Version string `json:"ver"` // current cas in little endian hex format of the current version on the version vector - MergeVersions []string `json:"mv,omitempty"` // list of merge versions in delta order. First elem will be full hex version, rest of items will be deltas calculated from the item above it - PreviousVersions []string `json:"pv,omitempty"` // list of previous versions in delta order. First elem will be full hex version, rest of items will be deltas calculated from the item above it -} +// constants for the json fields of the bucket version vector +const ( + jsonCvCAS = "cvCas" + jsonImportCAS = "importCAS" + jsonSourceID = "src" + jsonVersionCAS = "ver" + jsonMergeVersion = "mv" + jsonPreviousVersion = "pv" +) // NewHybridLogicalVector returns an initialised HybridLogicalVector. func NewHybridLogicalVector() HybridLogicalVector { @@ -595,91 +528,70 @@ func CreateEncodedSourceID(bucketUUID, clusterUUID string) (string, error) { } func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { - var cvCasByteArray []byte - var importCASBytes []byte - var vrsCasByteArray []byte + var cvCas string + var importCAS string + var vrsCas string + + bucketHLV := make(map[string]interface{}) if hlv.CurrentVersionCAS != 0 { - cvCasByteArray = base.Uint64CASToLittleEndianHex(hlv.CurrentVersionCAS) + cvCas = base.CasToString(hlv.CurrentVersionCAS) + bucketHLV[jsonCvCAS] = cvCas } if hlv.ImportCAS != 0 { - importCASBytes = base.Uint64CASToLittleEndianHex(hlv.ImportCAS) - } - if hlv.Version != 0 { - vrsCasByteArray = base.Uint64CASToLittleEndianHex(hlv.Version) + importCAS = base.CasToString(hlv.ImportCAS) + bucketHLV[jsonImportCAS] = importCAS } + vrsCas = base.CasToString(hlv.Version) + bucketHLV[jsonVersionCAS] = vrsCas + bucketHLV[jsonSourceID] = hlv.SourceID - pvPersistedFormat, err := convertMapToPersistedFormat(hlv.PreviousVersions) - if err != nil { - return nil, err + pvPersistedFormat := VersionsToDeltas(hlv.PreviousVersions) + if len(pvPersistedFormat) > 0 { + bucketHLV[jsonPreviousVersion] = pvPersistedFormat } - mvPersistedFormat, err := convertMapToPersistedFormat(hlv.MergeVersions) - if err != nil { - return nil, err + mvPersistedFormat := VersionsToDeltas(hlv.MergeVersions) + if len(mvPersistedFormat) > 0 { + bucketHLV[jsonMergeVersion] = mvPersistedFormat } - bucketVector := BucketVector{ - CurrentVersionCAS: string(cvCasByteArray), - ImportCAS: string(importCASBytes), - Version: string(vrsCasByteArray), - SourceID: hlv.SourceID, - MergeVersions: mvPersistedFormat, - PreviousVersions: pvPersistedFormat, - } - - return base.JSONMarshal(&bucketVector) + return base.JSONMarshal(&bucketHLV) } func (hlv *HybridLogicalVector) UnmarshalJSON(inputjson []byte) error { - persistedJSON := BucketVector{} - err := base.JSONUnmarshal(inputjson, &persistedJSON) + type BucketVector struct { + CurrentVersionCAS string `json:"cvCas,omitempty"` + ImportCAS string `json:"importCAS,omitempty"` + SourceID string `json:"src"` + Version string `json:"ver"` + PV *[]string `json:"pv,omitempty"` + MV *[]string `json:"mv,omitempty"` + } + var bucketDeltas BucketVector + err := base.JSONUnmarshal(inputjson, &bucketDeltas) if err != nil { return err } - // convert the data to in memory format - hlv.convertPersistedHLVToInMemoryHLV(persistedJSON) - return nil -} - -func (hlv *HybridLogicalVector) convertPersistedHLVToInMemoryHLV(persistedJSON BucketVector) { - hlv.CurrentVersionCAS = base.HexCasToUint64(persistedJSON.CurrentVersionCAS) - if persistedJSON.ImportCAS != "" { - hlv.ImportCAS = base.HexCasToUint64(persistedJSON.ImportCAS) + if bucketDeltas.CurrentVersionCAS != "" { + hlv.CurrentVersionCAS = base.HexCasToUint64(bucketDeltas.CurrentVersionCAS) } - hlv.SourceID = persistedJSON.SourceID - // convert the hex cas to uint64 cas - hlv.Version = base.HexCasToUint64(persistedJSON.Version) - // convert the maps form persisted format to the in memory format - hlv.PreviousVersions = convertMapToInMemoryFormat(persistedJSON.PreviousVersions) - hlv.MergeVersions = convertMapToInMemoryFormat(persistedJSON.MergeVersions) -} - -// convertMapToPersistedFormat will convert in memory map of previous versions or merge versions into the persisted format map -func convertMapToPersistedFormat(memoryMap map[string]uint64) (map[string]string, error) { - if memoryMap == nil { - return nil, nil + if bucketDeltas.ImportCAS != "" { + hlv.ImportCAS = base.HexCasToUint64(bucketDeltas.ImportCAS) } - returnedMap := make(map[string]string) - var persistedCAS string - for source, cas := range memoryMap { - casByteArray := base.Uint64CASToLittleEndianHex(cas) - persistedCAS = string(casByteArray) - // remove the leading '0x' from the CAS value - persistedCAS = persistedCAS[2:] - returnedMap[source] = persistedCAS - } - return returnedMap, nil -} - -// convertMapToInMemoryFormat will convert the persisted format map to an in memory format of that map. -// Used for previous versions and merge versions maps on HLV -func convertMapToInMemoryFormat(persistedMap map[string]string) map[string]uint64 { - if persistedMap == nil { - return nil + hlv.SourceID = bucketDeltas.SourceID + hlv.Version = base.HexCasToUint64(bucketDeltas.Version) + if bucketDeltas.PV != nil { + prevVersion, err := PersistedDeltasToMap(*bucketDeltas.PV) + if err != nil { + return err + } + hlv.PreviousVersions = prevVersion } - returnedMap := make(map[string]uint64) - // convert each CAS entry from little endian hex to Uint64 - for key, value := range persistedMap { - returnedMap[key] = base.HexCasToUint64(value) + if bucketDeltas.MV != nil { + mergeVersion, err := PersistedDeltasToMap(*bucketDeltas.MV) + if err != nil { + return err + } + hlv.MergeVersions = mergeVersion } - return returnedMap + return nil } diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index fb28559e24..f1e67a5f37 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -471,7 +471,6 @@ func TestParseCBLVersion(t *testing.T) { // the corresponding element in the original pvMap // - Do the same as above but for nil maps func TestVersionDeltaCalculation(t *testing.T) { - ctx := base.TestCtx(t) src1 := "src1" src2 := "src2" src3 := "src3" @@ -480,11 +479,11 @@ func TestVersionDeltaCalculation(t *testing.T) { timeNow := time.Now().UnixNano() // make some version deltas - v1 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) - v2 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) - v3 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) - v4 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) - v5 := base.CasToString(uint64(timeNow - rand.Int64N(1000000000000))) + v1 := uint64(timeNow - rand.Int64N(1000000000000)) + v2 := uint64(timeNow - rand.Int64N(1000000000000)) + v3 := uint64(timeNow - rand.Int64N(1000000000000)) + v4 := uint64(timeNow - rand.Int64N(1000000000000)) + v5 := uint64(timeNow - rand.Int64N(1000000000000)) // make map of source to version pvMap := make(HLVVersions) @@ -510,11 +509,12 @@ func TestVersionDeltaCalculation(t *testing.T) { expCas := hlv.CurrentVersionCAS // convert hlv to persisted format - vvXattr, err := ConstructXattrFromHlv(&hlv) + vvXattr, err := base.JSONMarshal(&hlv) require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err := ParseHLVFields(ctx, vvXattr) + memHLV := NewHybridLogicalVector() + err = base.JSONUnmarshal(vvXattr, &memHLV) require.NoError(t, err) assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) @@ -527,6 +527,7 @@ func TestVersionDeltaCalculation(t *testing.T) { assert.Equal(t, expSrc, memHLV.SourceID) assert.Equal(t, expVal, memHLV.Version) assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.MergeVersions, 0) // test hlv with nil merge versions and nil previous versions to test panic safe pvMap = nil @@ -535,12 +536,15 @@ func TestVersionDeltaCalculation(t *testing.T) { hlv2.MergeVersions = nil deltas = VersionDeltas(pvMap) assert.Nil(t, deltas) + // construct byte array from hlv - vvXattr, err = ConstructXattrFromHlv(&hlv2) + vvXattr, err = base.JSONMarshal(&hlv2) require.NoError(t, err) // convert the bytes back to an in memory format of hlv - memHLV, err = ParseHLVFields(ctx, vvXattr) + memHLV = HybridLogicalVector{} + err = base.JSONUnmarshal(vvXattr, &memHLV) require.NoError(t, err) + // assert in memory hlv is as expected assert.Equal(t, expSrc, memHLV.SourceID) assert.Equal(t, expVal, memHLV.Version) diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go index 1edd02ace4..ae4c2631ed 100644 --- a/db/utilities_hlv_testing.go +++ b/db/utilities_hlv_testing.go @@ -65,11 +65,11 @@ func (h *HLVAgent) InsertWithHLV(ctx context.Context, key string) (casOut uint64 // UpdateWithHLV will update and existing doc in bucket mocking write from another hlv aware peer func (h *HLVAgent) UpdateWithHLV(ctx context.Context, key string, inputCas uint64, hlv *HybridLogicalVector) (casOut uint64) { - err := hlv.AddVersion(CreateVersion(h.Source, hlvExpandMacroCASValue)) + err := hlv.AddVersion(CreateVersion(h.Source, expandMacroCASValueUint64)) require.NoError(h.t, err) - hlv.CurrentVersionCAS = hlvExpandMacroCASValue + hlv.CurrentVersionCAS = expandMacroCASValueUint64 - vvXattr, err := ConstructXattrFromHlv(hlv) + vvXattr, err := hlv.MarshalJSON() require.NoError(h.t, err) mutateInOpts := &sgbucket.MutateInOptions{ MacroExpansion: hlv.computeMacroExpansions(), diff --git a/rest/api_test.go b/rest/api_test.go index 4ef8cc0bf1..bc554c10fe 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -2819,7 +2819,7 @@ func TestPvDeltaReadAndWrite(t *testing.T) { hlvHelper := db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") existingHLVKey := docID cas := hlvHelper.InsertWithHLV(ctx, existingHLVKey) - encodedCASV1 := db.EncodeValue(cas) + casV1 := cas encodedSourceV1 := db.EncodeSource(otherSource) // force import of this write @@ -2829,20 +2829,20 @@ func TestPvDeltaReadAndWrite(t *testing.T) { version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"new": "update!"}) newDoc, _, err := collection.GetDocWithXattrs(ctx, existingHLVKey, db.DocUnmarshalAll) require.NoError(t, err) - encodedCASV2 := db.EncodeValue(newDoc.Cas) + casV2 := newDoc.Cas encodedSourceV2 := testSource // assert that we have a prev CV drop to pv and a new CV pair, assert pv values are as expected after delta conversions assert.Equal(t, testSource, newDoc.HLV.SourceID) assert.Equal(t, version2.CV.Value, newDoc.HLV.Version) assert.Len(t, newDoc.HLV.PreviousVersions, 1) - assert.Equal(t, encodedCASV1, newDoc.HLV.PreviousVersions[encodedSourceV1]) + assert.Equal(t, casV1, newDoc.HLV.PreviousVersions[encodedSourceV1]) otherSource = "diffSource" hlvHelper = db.NewHLVAgent(t, rt.GetSingleDataStore(), otherSource, "_vv") cas = hlvHelper.UpdateWithHLV(ctx, existingHLVKey, newDoc.Cas, newDoc.HLV) encodedSourceV3 := db.EncodeSource(otherSource) - encodedCASV3 := db.EncodeValue(cas) + casV3 := cas // import and get raw doc _, _ = rt.GetDoc(docID) @@ -2851,10 +2851,10 @@ func TestPvDeltaReadAndWrite(t *testing.T) { // assert that we have two entries in previous versions, and they are correctly converted from deltas back to full value assert.Equal(t, encodedSourceV3, bucketDoc.HLV.SourceID) - assert.Equal(t, encodedCASV3, bucketDoc.HLV.Version) + assert.Equal(t, casV3, bucketDoc.HLV.Version) assert.Len(t, bucketDoc.HLV.PreviousVersions, 2) - assert.Equal(t, encodedCASV1, bucketDoc.HLV.PreviousVersions[encodedSourceV1]) - assert.Equal(t, encodedCASV2, bucketDoc.HLV.PreviousVersions[encodedSourceV2]) + assert.Equal(t, casV1, bucketDoc.HLV.PreviousVersions[encodedSourceV1]) + assert.Equal(t, casV2, bucketDoc.HLV.PreviousVersions[encodedSourceV2]) } // TestPutDocUpdateVersionVector: From 3d3c5103c266149649b77df4825a90aff1aea937 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Fri, 4 Oct 2024 11:47:13 +0100 Subject: [PATCH 17/19] remove unused param --- db/change_cache.go | 2 +- db/document.go | 2 +- db/document_test.go | 4 ++-- db/import_listener.go | 2 +- rest/importuserxattrtest/import_test.go | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/db/change_cache.go b/db/change_cache.go index 6bc0a0489c..0e960e69bf 100644 --- a/db/change_cache.go +++ b/db/change_cache.go @@ -397,7 +397,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent) { } // First unmarshal the doc (just its metadata, to save time/memory): - syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(ctx, docJSON, event.DataType, collection.userXattrKey(), false) + syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.userXattrKey(), false) if err != nil { // Avoid log noise related to failed unmarshaling of binary documents. if event.DataType != base.MemcachedDataTypeRaw { diff --git a/db/document.go b/db/document.go index 385f7e0f6c..577c1dd304 100644 --- a/db/document.go +++ b/db/document.go @@ -463,7 +463,7 @@ func UnmarshalDocumentSyncData(data []byte, needHistory bool) (*SyncData, error) // Returns the raw body, in case it's needed for import. // TODO: Using a pool of unmarshal workers may help prevent memory spikes under load -func UnmarshalDocumentSyncDataFromFeed(ctx context.Context, data []byte, dataType uint8, userXattrKey string, needHistory bool) (result *SyncData, rawBody []byte, rawXattrs map[string][]byte, err error) { +func UnmarshalDocumentSyncDataFromFeed(data []byte, dataType uint8, userXattrKey string, needHistory bool) (result *SyncData, rawBody []byte, rawXattrs map[string][]byte, err error) { var body []byte // If xattr datatype flag is set, data includes both xattrs and document body. Check for presence of sync xattr. diff --git a/db/document_test.go b/db/document_test.go index 64c92d91c6..79dbced42a 100644 --- a/db/document_test.go +++ b/db/document_test.go @@ -494,7 +494,7 @@ func TestDCPDecodeValue(t *testing.T) { require.Nil(t, xattrs) } // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), test.body, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(test.body, base.MemcachedDataTypeXattr, "", false) require.ErrorIs(t, err, test.expectedErr) if test.expectedSyncXattr != nil { require.NotNil(t, result) @@ -519,7 +519,7 @@ func TestInvalidXattrStreamEmptyBody(t *testing.T) { require.Empty(t, xattrs) // UnmarshalDocumentSyncData wraps DecodeValueWithXattrs - result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), inputStream, base.MemcachedDataTypeXattr, "", false) + result, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(inputStream, base.MemcachedDataTypeXattr, "", false) require.Error(t, err) // unexpected end of JSON input require.Nil(t, result) require.Equal(t, emptyBody, rawBody) diff --git a/db/import_listener.go b/db/import_listener.go index 7104c5e2f5..a92c7b72ec 100644 --- a/db/import_listener.go +++ b/db/import_listener.go @@ -168,7 +168,7 @@ func (il *importListener) ProcessFeedEvent(event sgbucket.FeedEvent) (shouldPers } func (il *importListener) ImportFeedEvent(ctx context.Context, collection *DatabaseCollectionWithUser, event sgbucket.FeedEvent) { - syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(ctx, event.Value, event.DataType, collection.userXattrKey(), false) + syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(event.Value, event.DataType, collection.userXattrKey(), false) if err != nil { if errors.Is(err, sgbucket.ErrEmptyMetadata) { base.WarnfCtx(ctx, "Unexpected empty metadata when processing feed event. docid: %s opcode: %v datatype:%v", base.UD(event.Key), event.Opcode, event.DataType) diff --git a/rest/importuserxattrtest/import_test.go b/rest/importuserxattrtest/import_test.go index 6b419d0809..6e40678f0a 100644 --- a/rest/importuserxattrtest/import_test.go +++ b/rest/importuserxattrtest/import_test.go @@ -412,7 +412,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value := sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err := db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) require.NoError(t, err) assert.Equal(t, syncXattr, string(rawXattrs[base.SyncXattrName])) assert.Equal(t, uint64(200), syncData.Sequence) @@ -425,7 +425,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { } value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) @@ -436,7 +436,7 @@ func TestUnmarshalDocFromImportFeed(t *testing.T) { xattrs = []sgbucket.Xattr{} value = sgbucket.EncodeValueWithXattrs(body, xattrs...) - syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(base.TestCtx(t), value, 5, userXattrKey, false) + syncData, rawBody, rawXattrs, err = db.UnmarshalDocumentSyncDataFromFeed(value, 5, userXattrKey, false) require.NoError(t, err) assert.Nil(t, syncData) assert.Nil(t, rawXattrs[base.SyncXattrName]) From a0a346ad82cb531ee9c1737d04c6e63e6922012b Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Tue, 8 Oct 2024 12:05:34 +0100 Subject: [PATCH 18/19] updated to add new test cases and small changes --- base/util.go | 17 +++-------- base/util_test.go | 7 +++++ db/hybrid_logical_vector.go | 51 ++++++++++++++++++-------------- db/hybrid_logical_vector_test.go | 28 ++++++++++++++++++ 4 files changed, 68 insertions(+), 35 deletions(-) diff --git a/base/util.go b/base/util.go index 94896e0551..b2039afcd4 100644 --- a/base/util.go +++ b/base/util.go @@ -1021,9 +1021,6 @@ func HexCasToUint64(cas string) uint64 { // HexCasToUint64ForDelta will convert hex cas to uint64 accounting for any stripped zeros in delta calculation func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { var decoded []byte - if len(casByte) <= 2 { - return 0, fmt.Errorf("hex value is too short") - } // as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length if len(casByte)%2 != 0 { @@ -1055,14 +1052,16 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { } // Uint64ToLittleEndianHexAndStripZeros will convert a uint64 type to little endian hex, stripping any zeros off the end +// + stripping 0x from start func Uint64ToLittleEndianHexAndStripZeros(cas uint64) string { - hexCas := Uint64CASToLittleEndianHexNo0x(cas) + hexCas := Uint64CASToLittleEndianHex(cas) i := len(hexCas) - 1 for i > 2 && hexCas[i] == '0' { i-- } - return string(hexCas[:i+1]) + // strip 0x from start + return string(hexCas[2 : i+1]) } func HexToBase64(s string) ([]byte, error) { @@ -1075,14 +1074,6 @@ func HexToBase64(s string) ([]byte, error) { return encoded, nil } -func Uint64CASToLittleEndianHexNo0x(cas uint64) []byte { - littleEndian := make([]byte, 8) - binary.LittleEndian.PutUint64(littleEndian, cas) - encodedArray := make([]byte, hex.EncodedLen(8)) - _ = hex.Encode(encodedArray, littleEndian) - return encodedArray -} - func CasToString(cas uint64) string { return string(Uint64CASToLittleEndianHex(cas)) } diff --git a/base/util_test.go b/base/util_test.go index 51c40c65e8..97d67bd315 100644 --- a/base/util_test.go +++ b/base/util_test.go @@ -1764,4 +1764,11 @@ func TestUint64CASToLittleEndianHexAndStripZeros(t *testing.T) { u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) require.NoError(t, err) assert.Equal(t, u64, u64Stripped) + + hexLE = "0xa500000000000000" + u64 = HexCasToUint64(hexLE) + hexLEStripped = Uint64ToLittleEndianHexAndStripZeros(u64) + u64Stripped, err = HexCasToUint64ForDelta([]byte(hexLEStripped)) + require.NoError(t, err) + assert.Equal(t, u64, u64Stripped) } diff --git a/db/hybrid_logical_vector.go b/db/hybrid_logical_vector.go index 614300adaf..e32bc862a8 100644 --- a/db/hybrid_logical_vector.go +++ b/db/hybrid_logical_vector.go @@ -42,7 +42,7 @@ func (vde VersionsDeltas) Swap(i, j int) { func (vde VersionsDeltas) Less(i, j int) bool { if vde[i].Value == vde[j].Value { - return vde[i].SourceID < vde[j].SourceID + return false } return vde[i].Value < vde[j].Value } @@ -58,6 +58,11 @@ func VersionDeltas(versions map[string]uint64) VersionsDeltas { vdm = append(vdm, CreateVersion(src, vrs)) } + // return early for single entry + if len(vdm) == 1 { + return vdm + } + // sort the list sort.Sort(vdm) @@ -77,10 +82,7 @@ func VersionsToDeltas(m map[string]uint64) []string { var vrsList []string deltas := VersionDeltas(m) for _, delta := range deltas { - key := delta.SourceID - val := delta.Value - encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(val) - listItem := encodedVal + "@" + key + listItem := delta.StringForVersionDelta() vrsList = append(vrsList, listItem) } @@ -142,6 +144,13 @@ func (v Version) String() string { return strconv.FormatUint(v.Value, 16) + "@" + v.SourceID } +// StringForVersionDelta will take a version struct and convert the value to delta format +// (encoding it to LE hex, stripping any 0's off the end and stripping leading 0x) +func (v Version) StringForVersionDelta() string { + encodedVal := base.Uint64ToLittleEndianHexAndStripZeros(v.Value) + return encodedVal + "@" + v.SourceID +} + // ExtractCurrentVersionFromHLV will take the current version form the HLV struct and return it in the Version struct func (hlv *HybridLogicalVector) ExtractCurrentVersionFromHLV() *Version { src, vrs := hlv.GetCurrentVersion() @@ -159,16 +168,6 @@ type HybridLogicalVector struct { PreviousVersions HLVVersions // map of previous versions for fast efficient lookup } -// constants for the json fields of the bucket version vector -const ( - jsonCvCAS = "cvCas" - jsonImportCAS = "importCAS" - jsonSourceID = "src" - jsonVersionCAS = "ver" - jsonMergeVersion = "mv" - jsonPreviousVersion = "pv" -) - // NewHybridLogicalVector returns an initialised HybridLogicalVector. func NewHybridLogicalVector() HybridLogicalVector { return HybridLogicalVector{ @@ -528,30 +527,38 @@ func CreateEncodedSourceID(bucketUUID, clusterUUID string) (string, error) { } func (hlv HybridLogicalVector) MarshalJSON() ([]byte, error) { + type BucketVector struct { + CurrentVersionCAS string `json:"cvCas,omitempty"` + ImportCAS string `json:"importCAS,omitempty"` + SourceID string `json:"src"` + Version string `json:"ver"` + PV *[]string `json:"pv,omitempty"` + MV *[]string `json:"mv,omitempty"` + } var cvCas string var importCAS string var vrsCas string - bucketHLV := make(map[string]interface{}) + var bucketHLV = BucketVector{} if hlv.CurrentVersionCAS != 0 { cvCas = base.CasToString(hlv.CurrentVersionCAS) - bucketHLV[jsonCvCAS] = cvCas + bucketHLV.CurrentVersionCAS = cvCas } if hlv.ImportCAS != 0 { importCAS = base.CasToString(hlv.ImportCAS) - bucketHLV[jsonImportCAS] = importCAS + bucketHLV.ImportCAS = importCAS } vrsCas = base.CasToString(hlv.Version) - bucketHLV[jsonVersionCAS] = vrsCas - bucketHLV[jsonSourceID] = hlv.SourceID + bucketHLV.Version = vrsCas + bucketHLV.SourceID = hlv.SourceID pvPersistedFormat := VersionsToDeltas(hlv.PreviousVersions) if len(pvPersistedFormat) > 0 { - bucketHLV[jsonPreviousVersion] = pvPersistedFormat + bucketHLV.PV = &pvPersistedFormat } mvPersistedFormat := VersionsToDeltas(hlv.MergeVersions) if len(mvPersistedFormat) > 0 { - bucketHLV[jsonMergeVersion] = mvPersistedFormat + bucketHLV.MV = &mvPersistedFormat } return base.JSONMarshal(&bucketHLV) diff --git a/db/hybrid_logical_vector_test.go b/db/hybrid_logical_vector_test.go index f1e67a5f37..a0fd0070a5 100644 --- a/db/hybrid_logical_vector_test.go +++ b/db/hybrid_logical_vector_test.go @@ -469,6 +469,7 @@ func TestParseCBLVersion(t *testing.T) { // - Create a test HLV and convert it to persisted format in bytes // - Convert this back to in memory format, assert each elem of in memory format previous versions map is the same as // the corresponding element in the original pvMap +// - Do the same for a pv map that will have two entries with the same version value // - Do the same as above but for nil maps func TestVersionDeltaCalculation(t *testing.T) { src1 := "src1" @@ -529,6 +530,33 @@ func TestVersionDeltaCalculation(t *testing.T) { assert.Equal(t, expCas, memHLV.CurrentVersionCAS) assert.Len(t, memHLV.MergeVersions, 0) + // test hlv with two pv version entries that are equal to each other + hlv = createHLVForTest(t, inputHLVA) + // make src3 have the same version value as src2 + pvMap[src3] = pvMap[src2] + hlv.PreviousVersions = pvMap + + // convert hlv to persisted format + vvXattr, err = base.JSONMarshal(&hlv) + require.NoError(t, err) + + // convert the bytes back to an in memory format of hlv + memHLV = NewHybridLogicalVector() + err = base.JSONUnmarshal(vvXattr, &memHLV) + require.NoError(t, err) + + assert.Equal(t, pvMap[src1], memHLV.PreviousVersions[src1]) + assert.Equal(t, pvMap[src2], memHLV.PreviousVersions[src2]) + assert.Equal(t, pvMap[src3], memHLV.PreviousVersions[src3]) + assert.Equal(t, pvMap[src4], memHLV.PreviousVersions[src4]) + assert.Equal(t, pvMap[src5], memHLV.PreviousVersions[src5]) + + // assert that the other elements are as expected + assert.Equal(t, expSrc, memHLV.SourceID) + assert.Equal(t, expVal, memHLV.Version) + assert.Equal(t, expCas, memHLV.CurrentVersionCAS) + assert.Len(t, memHLV.MergeVersions, 0) + // test hlv with nil merge versions and nil previous versions to test panic safe pvMap = nil hlv2 := createHLVForTest(t, inputHLVA) From 2b63f97d1e67d0b82d2cf2b76c7e6d776d4b24d3 Mon Sep 17 00:00:00 2001 From: Gregory Newman-Smith Date: Thu, 10 Oct 2024 09:45:19 +0100 Subject: [PATCH 19/19] small update --- base/util.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/base/util.go b/base/util.go index b2039afcd4..b48a802322 100644 --- a/base/util.go +++ b/base/util.go @@ -1024,10 +1024,7 @@ func HexCasToUint64ForDelta(casByte []byte) (uint64, error) { // as we strip any zeros off the end of the hex value for deltas, the input delta could be odd length if len(casByte)%2 != 0 { - evenHexLen := make([]byte, len(casByte), len(casByte)+1) - copy(evenHexLen, casByte) - evenHexLen = append(evenHexLen, '0') - casByte = evenHexLen + casByte = append(casByte, '0') } // create byte array for decoding into