From d32fb64d6901d42477f0220fc95d535ed513de8d Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 09:53:11 +0100 Subject: [PATCH 01/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20use=20C?= =?UTF-8?q?BOR=20base=20codec=20for=20indexed=20CRDT=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 72 +++++++++---------- src/json-crdt/codec/indexed/binary/Encoder.ts | 50 ++++++++----- .../indexed/binary/__tests__/codec.spec.ts | 13 ++++ .../codec/structural/binary/Encoder.ts | 68 +++++++++--------- 4 files changed, 112 insertions(+), 91 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 4ead91dddc..40811d3bbe 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -15,13 +15,18 @@ import {CrdtReader} from '../../../../json-crdt-patch/util/binary/CrdtReader'; import {IndexedFields, FieldName, IndexedNodeFields} from './types'; import {ITimestampStruct, IVectorClock, Timestamp, VectorClock} from '../../../../json-crdt-patch/clock'; import {Model, UNDEFINED} from '../../../model/Model'; -import {MsgPackDecoderFast} from '../../../../json-pack/msgpack'; +import {CborDecoderBase} from '../../../../json-pack/cbor/CborDecoderBase'; +import {CRDT_MAJOR} from '../../structural/binary/constants'; export class Decoder { - public readonly dec = new MsgPackDecoderFast(new CrdtReader()); + public readonly dec: CborDecoderBase; protected doc!: Model; protected clockTable?: ClockTable; + constructor(reader?: CrdtReader) { + this.dec = new CborDecoderBase(reader || new CrdtReader()); + } + public decode( fields: IndexedFields, ModelConstructor: new (clock: IVectorClock) => M = Model as unknown as new (clock: IVectorClock) => M, @@ -66,47 +71,36 @@ export class Decoder { protected decodeNode(id: ITimestampStruct): JsonNode { const reader = this.dec.reader; - const byte = reader.u8(); - if (byte <= 0b10001111) return this.cObj(id, byte & 0b1111); - else if (byte <= 0b10011111) return this.cArr(id, byte & 0b1111); - else if (byte <= 0b10111111) return this.cStr(id, byte & 0b11111); - else { - switch (byte) { - case 0xc4: - return this.cBin(id, reader.u8()); - case 0xc5: - return this.cBin(id, reader.u16()); - case 0xc6: - return this.cBin(id, reader.u32()); - case 0xd4: - return this.cConst(id); - case 0xd5: - return new ConNode(id, this.ts()); - case 0xd6: - return this.cVal(id); - case 0xde: - return this.cObj(id, reader.u16()); - case 0xdf: - return this.cObj(id, reader.u32()); - case 0xdc: - return this.cArr(id, reader.u16()); - case 0xdd: - return this.cArr(id, reader.u32()); - case 0xd9: - return this.cStr(id, reader.u8()); - case 0xda: - return this.cStr(id, reader.u16()); - case 0xdb: - return this.cStr(id, reader.u32()); - } + const octet = reader.u8(); + const major = octet >> 5; + const minor = octet & 0b11111; + const length = minor < 24 ? minor : minor === 24 ? reader.u8() : minor === 25 ? reader.u16() : reader.u32(); + switch (major) { + case CRDT_MAJOR.CON: + return this.decodeCon(id, length); + // case CRDT_MAJOR.VAL: + // return this.cVal(id); + // case CRDT_MAJOR.VEC: + // return this.cVec(id, length); + // case CRDT_MAJOR.OBJ: + // return this.cObj(id, length); + // case CRDT_MAJOR.STR: + // return this.cStr(id, length); + // case CRDT_MAJOR.BIN: + // return this.cBin(id, length); + // case CRDT_MAJOR.ARR: + // return this.cArr(id, length); } - return UNDEFINED; } - public cConst(id: ITimestampStruct): ConNode { - const val = this.dec.val(); - return new ConNode(id, val); + public decodeCon(id: ITimestampStruct, length: number): ConNode { + const doc = this.doc; + const decoder = this.dec; + const data = !length ? decoder.val() : this.ts(); + const node = new ConNode(id, data); + doc.index.set(id, node); + return node; } public cVal(id: ITimestampStruct): ValNode { diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 63074d3638..75411bb062 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -1,18 +1,23 @@ import {ITimestampStruct, Timestamp} from '../../../../json-crdt-patch/clock'; import {ClockTable} from '../../../../json-crdt-patch/codec/clock/ClockTable'; import {CrdtWriter} from '../../../../json-crdt-patch/util/binary/CrdtWriter'; -import {MsgPackEncoder} from '../../../../json-pack/msgpack'; +import {CborEncoder} from '../../../../json-pack/cbor/CborEncoder'; import {Model} from '../../../model'; import {ConNode, JsonNode, ValNode, ArrNode, BinNode, ObjNode, StrNode} from '../../../nodes'; +import {CRDT_MAJOR_OVERLAY} from '../../structural/binary/constants'; import {IndexedFields, FieldName} from './types'; const EMPTY = new Uint8Array(0); export class Encoder { + public readonly enc: CborEncoder; protected clockTable?: ClockTable; - public readonly enc = new MsgPackEncoder(new CrdtWriter()); protected model?: IndexedFields; + constructor(writer?: CrdtWriter) { + this.enc = new CborEncoder(writer || new CrdtWriter()); + } + public encode(doc: Model, clockTable: ClockTable = ClockTable.from(doc.clock)): IndexedFields { this.clockTable = clockTable; const writer = this.enc.writer; @@ -45,7 +50,7 @@ export class Encoder { public encodeNode(node: JsonNode): Uint8Array { if (node instanceof ValNode) return this.encodeVal(node); - else if (node instanceof ConNode) return this.encodeConst(node); + else if (node instanceof ConNode) return this.encodeCon(node); else if (node instanceof StrNode) return this.encodeStr(node); else if (node instanceof ObjNode) return this.encodeObj(node); else if (node instanceof ArrNode) return this.encodeArr(node); @@ -67,32 +72,41 @@ export class Encoder { return writer.flush(); } - public encodeConst(node: ConNode): Uint8Array { + protected writeTL(majorOverlay: CRDT_MAJOR_OVERLAY, length: number): void { + const writer = this.enc.writer; + if (length < 24) writer.u8(majorOverlay + length); + else if (length <= 0xff) writer.u16(((majorOverlay + 24) << 8) + length); + else if (length <= 0xffff) writer.u8u16(majorOverlay + 25, length); + else writer.u8u32(majorOverlay + 26, length); + } + + public encodeCon(node: ConNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; const val = node.val; writer.reset(); if (val instanceof Timestamp) { - writer.u8(0xd5); - this.ts(val); + this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1); + this.ts(val as Timestamp); } else { - writer.u8(0xd4); - encoder.writeAny(node.val); + this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0); + encoder.writeAny(val); } return writer.flush(); } public encodeStr(node: StrNode): Uint8Array { - const encoder = this.enc; - const writer = encoder.writer; - writer.reset(); - encoder.writeStrHdr(node.size()); - for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { - this.ts(chunk.id); - if (chunk.del) encoder.u32(chunk.span); - else encoder.encodeString(chunk.data!); - } - return writer.flush(); + throw new Error('TODO'); + // const encoder = this.enc; + // const writer = encoder.writer; + // writer.reset(); + // encoder.writeStrHdr(node.size()); + // for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { + // this.ts(chunk.id); + // if (chunk.del) encoder.u32(chunk.span); + // else encoder.encodeString(chunk.data!); + // } + // return writer.flush(); } public encodeBin(node: BinNode): Uint8Array { diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index 6eab244b55..82d3ed480e 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -70,3 +70,16 @@ test('can encode ID as const value', () => { expect(ts).toBeInstanceOf(Timestamp); expect(equal(ts, new Timestamp(model.clock.sid, 2))).toBe(true); }); + +describe.only('basic types', () => { + test('con', () => { + const model = Model.withLogicalClock(); + model.api.root(konst(123)); + const encoded = encoder.encode(model); + console.log(model + ''); + console.log(encoded); + const decoded = decoder.decode(encoded); + const view = decoded.view(); + expect(view).toBe(123); + }); +}); diff --git a/src/json-crdt/codec/structural/binary/Encoder.ts b/src/json-crdt/codec/structural/binary/Encoder.ts index 5555468cea..00b60449cb 100644 --- a/src/json-crdt/codec/structural/binary/Encoder.ts +++ b/src/json-crdt/codec/structural/binary/Encoder.ts @@ -12,8 +12,8 @@ export class Encoder extends CborEncoder { protected time: number = 0; protected doc!: Model; - constructor() { - super(new CrdtWriter()); + constructor(writer?: CrdtWriter) { + super(writer || new CrdtWriter()); } public encode(doc: Model): Uint8Array { @@ -96,6 +96,24 @@ export class Encoder extends CborEncoder { else if (node instanceof BinNode) this.cBin(node); } + protected cCon(node: ConNode): void { + const val = node.val; + this.ts(node.id); + if (val instanceof Timestamp) { + this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1); + this.ts(val as Timestamp); + } else { + this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0); + this.writeAny(val); + } + } + + protected cVal(node: ValNode): void { + this.ts(node.id); + this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); + this.cNode(node.node()); + } + protected cObj(node: ObjNode): void { this.ts(node.id); const keys = node.keys; @@ -121,23 +139,6 @@ export class Encoder extends CborEncoder { } } - protected cArr(node: ArrNode): void { - const ts = this.ts; - const writer = this.writer; - ts(node.id); - this.writeTL(CRDT_MAJOR_OVERLAY.ARR, node.count); - const index = this.doc.index; - for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { - const span = chunk.span; - const deleted = chunk.del; - writer.b1vu28(deleted, span); - ts(chunk.id); - if (deleted) continue; - const nodes = chunk.data!; - for (let i = 0; i < span; i++) this.cNode(index.get(nodes[i])!); - } - } - protected cStr(node: StrNode): void { const ts = this.ts; const writer = this.writer; @@ -167,21 +168,20 @@ export class Encoder extends CborEncoder { } } - protected cVal(node: ValNode): void { - this.ts(node.id); - this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); - this.cNode(node.node()); - } - - protected cCon(node: ConNode): void { - const val = node.val; - this.ts(node.id); - if (val instanceof Timestamp) { - this.writeTL(CRDT_MAJOR_OVERLAY.CON, 1); - this.ts(val as Timestamp); - } else { - this.writeTL(CRDT_MAJOR_OVERLAY.CON, 0); - this.writeAny(val); + protected cArr(node: ArrNode): void { + const ts = this.ts; + const writer = this.writer; + ts(node.id); + this.writeTL(CRDT_MAJOR_OVERLAY.ARR, node.count); + const index = this.doc.index; + for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { + const span = chunk.span; + const deleted = chunk.del; + writer.b1vu28(deleted, span); + ts(chunk.id); + if (deleted) continue; + const nodes = chunk.data!; + for (let i = 0; i < span; i++) this.cNode(index.get(nodes[i])!); } } } From 83a6c9902c16d7506d917439a55be245e6f5c5ca Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 09:55:07 +0100 Subject: [PATCH 02/13] =?UTF-8?q?refactor(json-crdt):=20=F0=9F=92=A1=20use?= =?UTF-8?q?=20namespace=20imports=20for=20nodes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 51 ++++++++----------- .../codec/structural/binary/Encoder.ts | 34 ++++++------- 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 40811d3bbe..fe697cba51 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -1,15 +1,4 @@ -import { - ConNode, - JsonNode, - ValNode, - ArrNode, - ArrChunk, - BinNode, - BinChunk, - ObjNode, - StrNode, - StrChunk, -} from '../../../nodes'; +import * as nodes from '../../../nodes'; import {ClockTable} from '../../../../json-crdt-patch/codec/clock/ClockTable'; import {CrdtReader} from '../../../../json-crdt-patch/util/binary/CrdtReader'; import {IndexedFields, FieldName, IndexedNodeFields} from './types'; @@ -69,7 +58,7 @@ export class Decoder { return new Timestamp(this.clockTable!.byIdx[sessionIndex].sid, timeDiff); } - protected decodeNode(id: ITimestampStruct): JsonNode { + protected decodeNode(id: ITimestampStruct): nodes.JsonNode { const reader = this.dec.reader; const octet = reader.u8(); const major = octet >> 5; @@ -94,23 +83,23 @@ export class Decoder { return UNDEFINED; } - public decodeCon(id: ITimestampStruct, length: number): ConNode { + public decodeCon(id: ITimestampStruct, length: number): nodes.ConNode { const doc = this.doc; const decoder = this.dec; const data = !length ? decoder.val() : this.ts(); - const node = new ConNode(id, data); + const node = new nodes.ConNode(id, data); doc.index.set(id, node); return node; } - public cVal(id: ITimestampStruct): ValNode { + public cVal(id: ITimestampStruct): nodes.ValNode { const val = this.ts(); - return new ValNode(this.doc, id, val); + return new nodes.ValNode(this.doc, id, val); } - public cObj(id: ITimestampStruct, length: number): ObjNode { + public cObj(id: ITimestampStruct, length: number): nodes.ObjNode { const decoder = this.dec; - const obj = new ObjNode(this.doc, id); + const obj = new nodes.ObjNode(this.doc, id); const keys = obj.keys; for (let i = 0; i < length; i++) { const key = String(decoder.val()); @@ -120,44 +109,44 @@ export class Decoder { return obj; } - protected cStr(id: ITimestampStruct, length: number): StrNode { + protected cStr(id: ITimestampStruct, length: number): nodes.StrNode { const decoder = this.dec; - const node = new StrNode(id); + const node = new nodes.StrNode(id); node.ingest(length, () => { const chunkId = this.ts(); const val = decoder.val(); - if (typeof val === 'number') return new StrChunk(chunkId, val, ''); + if (typeof val === 'number') return new nodes.StrChunk(chunkId, val, ''); const data = String(val); - return new StrChunk(chunkId, data.length, data); + return new nodes.StrChunk(chunkId, data.length, data); }); return node; } - protected cBin(id: ITimestampStruct, length: number): BinNode { + protected cBin(id: ITimestampStruct, length: number): nodes.BinNode { const decoder = this.dec; const reader = decoder.reader; - const node = new BinNode(id); + const node = new nodes.BinNode(id); node.ingest(length, () => { const chunkId = this.ts(); const [deleted, length] = reader.b1vu28(); - if (deleted) return new BinChunk(chunkId, length, undefined); + if (deleted) return new nodes.BinChunk(chunkId, length, undefined); const data = reader.buf(length); - return new BinChunk(chunkId, length, data); + return new nodes.BinChunk(chunkId, length, data); }); return node; } - protected cArr(id: ITimestampStruct, length: number): ArrNode { + protected cArr(id: ITimestampStruct, length: number): nodes.ArrNode { const decoder = this.dec; const reader = decoder.reader; - const node = new ArrNode(this.doc, id); + const node = new nodes.ArrNode(this.doc, id); node.ingest(length, () => { const chunkId = this.ts(); const [deleted, length] = reader.b1vu28(); - if (deleted) return new ArrChunk(chunkId, length, undefined); + if (deleted) return new nodes.ArrChunk(chunkId, length, undefined); const data: ITimestampStruct[] = []; for (let i = 0; i < length; i++) data.push(this.ts()); - return new ArrChunk(chunkId, length, data); + return new nodes.ArrChunk(chunkId, length, data); }); return node; } diff --git a/src/json-crdt/codec/structural/binary/Encoder.ts b/src/json-crdt/codec/structural/binary/Encoder.ts index 00b60449cb..596c22ddb1 100644 --- a/src/json-crdt/codec/structural/binary/Encoder.ts +++ b/src/json-crdt/codec/structural/binary/Encoder.ts @@ -1,4 +1,4 @@ -import {ConNode, RootNode, JsonNode, ValNode, VecNode, ArrNode, BinNode, ObjNode, StrNode} from '../../../nodes'; +import * as nodes from '../../../nodes'; import {ClockEncoder} from '../../../../json-crdt-patch/codec/clock/ClockEncoder'; import {CrdtWriter} from '../../../../json-crdt-patch/util/binary/CrdtWriter'; import {ITimestampStruct, Timestamp} from '../../../../json-crdt-patch/clock'; @@ -71,7 +71,7 @@ export class Encoder extends CborEncoder { protected ts: (ts: ITimestampStruct) => void = this.tsLogical; - protected cRoot(root: RootNode): void { + protected cRoot(root: nodes.RootNode): void { const val = root.val; if (val.sid === SESSION.SYSTEM) this.writer.u8(0); else this.cNode(root.node()); @@ -85,18 +85,18 @@ export class Encoder extends CborEncoder { else writer.u8u32(majorOverlay + 26, length); } - protected cNode(node: JsonNode): void { + protected cNode(node: nodes.JsonNode): void { // TODO: PERF: use a switch? - if (node instanceof ConNode) this.cCon(node); - else if (node instanceof ValNode) this.cVal(node); - else if (node instanceof StrNode) this.cStr(node); - else if (node instanceof ObjNode) this.cObj(node); - else if (node instanceof VecNode) this.cVec(node); - else if (node instanceof ArrNode) this.cArr(node); - else if (node instanceof BinNode) this.cBin(node); + if (node instanceof nodes.ConNode) this.cCon(node); + else if (node instanceof nodes.ValNode) this.cVal(node); + else if (node instanceof nodes.StrNode) this.cStr(node); + else if (node instanceof nodes.ObjNode) this.cObj(node); + else if (node instanceof nodes.VecNode) this.cVec(node); + else if (node instanceof nodes.ArrNode) this.cArr(node); + else if (node instanceof nodes.BinNode) this.cBin(node); } - protected cCon(node: ConNode): void { + protected cCon(node: nodes.ConNode): void { const val = node.val; this.ts(node.id); if (val instanceof Timestamp) { @@ -108,13 +108,13 @@ export class Encoder extends CborEncoder { } } - protected cVal(node: ValNode): void { + protected cVal(node: nodes.ValNode): void { this.ts(node.id); this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); this.cNode(node.node()); } - protected cObj(node: ObjNode): void { + protected cObj(node: nodes.ObjNode): void { this.ts(node.id); const keys = node.keys; this.writeTL(CRDT_MAJOR_OVERLAY.OBJ, keys.size); @@ -126,7 +126,7 @@ export class Encoder extends CborEncoder { this.cNode(this.doc.index.get(val)!); }; - protected cVec(node: VecNode): void { + protected cVec(node: nodes.VecNode): void { const elements = node.elements; const length = elements.length; this.ts(node.id); @@ -139,7 +139,7 @@ export class Encoder extends CborEncoder { } } - protected cStr(node: StrNode): void { + protected cStr(node: nodes.StrNode): void { const ts = this.ts; const writer = this.writer; ts(node.id); @@ -153,7 +153,7 @@ export class Encoder extends CborEncoder { } } - protected cBin(node: BinNode): void { + protected cBin(node: nodes.BinNode): void { const ts = this.ts; const writer = this.writer; ts(node.id); @@ -168,7 +168,7 @@ export class Encoder extends CborEncoder { } } - protected cArr(node: ArrNode): void { + protected cArr(node: nodes.ArrNode): void { const ts = this.ts; const writer = this.writer; ts(node.id); From 43b2ee83d77a279d72ad3f7acbf1b3dbe863657a Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 10:06:10 +0100 Subject: [PATCH 03/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20s?= =?UTF-8?q?upport=20for=20"val"=20node=20type=20in=20indexed=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 20 ++++++++++--------- src/json-crdt/codec/indexed/binary/Encoder.ts | 18 ++++++++--------- .../indexed/binary/__tests__/codec.spec.ts | 20 +++++++++++++++++-- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index fe697cba51..701fdbbfec 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -41,14 +41,17 @@ export class Decoder { const rootValue = this.ts(); doc.root.set(rootValue); } - const docIndex = doc.index; - for (const field in fields) { + const index = doc.index; + const keys = Object.keys(fields); + const length = keys.length; + for (let i = 0; i < length; i++) { + const field = keys[i]; if (field.length < 3) continue; // Skip "c" and "r". const arr = fields[field as FieldName]; const id = clockTable.parseField(field as FieldName); reader.reset(arr); const node = this.decodeNode(id); - docIndex.set(node.id, node); + index.set(id, node); } return doc; } @@ -67,8 +70,8 @@ export class Decoder { switch (major) { case CRDT_MAJOR.CON: return this.decodeCon(id, length); - // case CRDT_MAJOR.VAL: - // return this.cVal(id); + case CRDT_MAJOR.VAL: + return this.decodeVal(id); // case CRDT_MAJOR.VEC: // return this.cVec(id, length); // case CRDT_MAJOR.OBJ: @@ -84,17 +87,16 @@ export class Decoder { } public decodeCon(id: ITimestampStruct, length: number): nodes.ConNode { - const doc = this.doc; const decoder = this.dec; const data = !length ? decoder.val() : this.ts(); const node = new nodes.ConNode(id, data); - doc.index.set(id, node); return node; } - public cVal(id: ITimestampStruct): nodes.ValNode { + public decodeVal(id: ITimestampStruct): nodes.ValNode { const val = this.ts(); - return new nodes.ValNode(this.doc, id, val); + const node = new nodes.ValNode(this.doc, id, val); + return node; } public cObj(id: ITimestampStruct, length: number): nodes.ObjNode { diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 75411bb062..89f528457c 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -63,15 +63,6 @@ export class Encoder { this.enc.writer.id(index, id.time); } - public encodeVal(node: ValNode): Uint8Array { - const writer = this.enc.writer; - const child = node.node(); - writer.reset(); - writer.u8(0xd6); - this.ts(child.id); - return writer.flush(); - } - protected writeTL(majorOverlay: CRDT_MAJOR_OVERLAY, length: number): void { const writer = this.enc.writer; if (length < 24) writer.u8(majorOverlay + length); @@ -95,6 +86,15 @@ export class Encoder { return writer.flush(); } + public encodeVal(node: ValNode): Uint8Array { + const writer = this.enc.writer; + const child = node.node(); + writer.reset(); + this.writeTL(CRDT_MAJOR_OVERLAY.VAL, 0); + this.ts(child.id); + return writer.flush(); + } + public encodeStr(node: StrNode): Uint8Array { throw new Error('TODO'); // const encoder = this.enc; diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index 82d3ed480e..d081679260 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -3,6 +3,7 @@ import {Encoder} from '../Encoder'; import {Decoder} from '../Decoder'; import {compare, equal, Timestamp, VectorClock} from '../../../../../json-crdt-patch/clock'; import {konst} from '../../../../../json-crdt-patch/builder/Konst'; +import {s} from '../../../../../json-crdt-patch'; const encoder = new Encoder(); const decoder = new Decoder(); @@ -76,10 +77,25 @@ describe.only('basic types', () => { const model = Model.withLogicalClock(); model.api.root(konst(123)); const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('val', () => { + const model = Model.withLogicalClock(); + model.setSchema(s.val(s.con(true))); + const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('obj', () => { + const model = Model.withLogicalClock(); + model.api.root({foo: null}); + const encoded = encoder.encode(model); console.log(model + ''); console.log(encoded); const decoded = decoder.decode(encoded); - const view = decoded.view(); - expect(view).toBe(123); + expect(decoded.view()).toStrictEqual(model.view()); }); }); From 9a8c165fb005f68ee12fa067d737827f71233d7a Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 10:10:06 +0100 Subject: [PATCH 04/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20"?= =?UTF-8?q?obj"=20node=20type=20support=20for=20indexed=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 8 ++--- src/json-crdt/codec/indexed/binary/Encoder.ts | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 701fdbbfec..7eafed9284 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -72,10 +72,10 @@ export class Decoder { return this.decodeCon(id, length); case CRDT_MAJOR.VAL: return this.decodeVal(id); + case CRDT_MAJOR.OBJ: + return this.decodeObj(id, length); // case CRDT_MAJOR.VEC: // return this.cVec(id, length); - // case CRDT_MAJOR.OBJ: - // return this.cObj(id, length); // case CRDT_MAJOR.STR: // return this.cStr(id, length); // case CRDT_MAJOR.BIN: @@ -99,12 +99,12 @@ export class Decoder { return node; } - public cObj(id: ITimestampStruct, length: number): nodes.ObjNode { + public decodeObj(id: ITimestampStruct, length: number): nodes.ObjNode { const decoder = this.dec; const obj = new nodes.ObjNode(this.doc, id); const keys = obj.keys; for (let i = 0; i < length; i++) { - const key = String(decoder.val()); + const key = decoder.val() + ''; const val = this.ts(); keys.set(key, val); } diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 89f528457c..01339bb602 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -95,6 +95,21 @@ export class Encoder { return writer.flush(); } + public encodeObj(node: ObjNode): Uint8Array { + const encoder = this.enc; + const writer = encoder.writer; + writer.reset(); + const keys = node.keys; + this.writeTL(CRDT_MAJOR_OVERLAY.OBJ, keys.size); + keys.forEach(this.onObjKey); + return writer.flush(); + } + + private readonly onObjKey = (value: ITimestampStruct, key: string) => { + this.enc.writeStr(key); + this.ts(value); + }; + public encodeStr(node: StrNode): Uint8Array { throw new Error('TODO'); // const encoder = this.enc; @@ -125,20 +140,6 @@ export class Encoder { return writer.flush(); } - public encodeObj(node: ObjNode): Uint8Array { - const encoder = this.enc; - const writer = encoder.writer; - writer.reset(); - encoder.writeObjHdr(node.keys.size); - node.keys.forEach(this.onObjectKey); - return writer.flush(); - } - - protected readonly onObjectKey = (value: ITimestampStruct, key: string) => { - this.enc.writeStr(key); - this.ts(value); - }; - public encodeArr(node: ArrNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; From ce662b5cff709ea5acf8122f75ee90d1770f6012 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 10:31:44 +0100 Subject: [PATCH 05/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20support?= =?UTF-8?q?=20"vec"=20node=20in=20indexed=20JSON=20CRDT=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 16 ++++++- src/json-crdt/codec/indexed/binary/Encoder.ts | 47 +++++++++++++------ .../indexed/binary/__tests__/codec.spec.ts | 8 ++++ .../codec/structural/binary/Decoder.ts | 4 +- src/json-crdt/nodes/vec/VecNode.ts | 2 +- 5 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 7eafed9284..480e06458e 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -74,8 +74,8 @@ export class Decoder { return this.decodeVal(id); case CRDT_MAJOR.OBJ: return this.decodeObj(id, length); - // case CRDT_MAJOR.VEC: - // return this.cVec(id, length); + case CRDT_MAJOR.VEC: + return this.decodeVec(id, length); // case CRDT_MAJOR.STR: // return this.cStr(id, length); // case CRDT_MAJOR.BIN: @@ -110,6 +110,18 @@ export class Decoder { } return obj; } + + public decodeVec(id: ITimestampStruct, length: number): nodes.VecNode { + const reader = this.dec.reader; + const node = new nodes.VecNode(this.doc, id); + const elements = node.elements; + for (let i = 0; i < length; i++) { + const octet = reader.u8(); + if (!octet) elements.push(undefined); + else elements.push(this.ts()); + } + return node; + } protected cStr(id: ITimestampStruct, length: number): nodes.StrNode { const decoder = this.dec; diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 01339bb602..6d3951eca1 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -3,7 +3,7 @@ import {ClockTable} from '../../../../json-crdt-patch/codec/clock/ClockTable'; import {CrdtWriter} from '../../../../json-crdt-patch/util/binary/CrdtWriter'; import {CborEncoder} from '../../../../json-pack/cbor/CborEncoder'; import {Model} from '../../../model'; -import {ConNode, JsonNode, ValNode, ArrNode, BinNode, ObjNode, StrNode} from '../../../nodes'; +import * as nodes from '../../../nodes'; import {CRDT_MAJOR_OVERLAY} from '../../structural/binary/constants'; import {IndexedFields, FieldName} from './types'; @@ -37,7 +37,7 @@ export class Encoder { return model; } - protected readonly onNode = (node: JsonNode) => { + protected readonly onNode = (node: nodes.JsonNode) => { const id = node.id; const sid = id.sid; const time = id.time; @@ -48,13 +48,14 @@ export class Encoder { model[field] = this.encodeNode(node); }; - public encodeNode(node: JsonNode): Uint8Array { - if (node instanceof ValNode) return this.encodeVal(node); - else if (node instanceof ConNode) return this.encodeCon(node); - else if (node instanceof StrNode) return this.encodeStr(node); - else if (node instanceof ObjNode) return this.encodeObj(node); - else if (node instanceof ArrNode) return this.encodeArr(node); - else if (node instanceof BinNode) return this.encodeBin(node); + public encodeNode(node: nodes.JsonNode): Uint8Array { + if (node instanceof nodes.ConNode) return this.encodeCon(node); + else if (node instanceof nodes.ValNode) return this.encodeVal(node); + else if (node instanceof nodes.ObjNode) return this.encodeObj(node); + else if (node instanceof nodes.VecNode) return this.encodeVec(node); + else if (node instanceof nodes.StrNode) return this.encodeStr(node); + else if (node instanceof nodes.BinNode) return this.encodeBin(node); + else if (node instanceof nodes.ArrNode) return this.encodeArr(node); else return EMPTY; } @@ -71,7 +72,7 @@ export class Encoder { else writer.u8u32(majorOverlay + 26, length); } - public encodeCon(node: ConNode): Uint8Array { + public encodeCon(node: nodes.ConNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; const val = node.val; @@ -86,7 +87,7 @@ export class Encoder { return writer.flush(); } - public encodeVal(node: ValNode): Uint8Array { + public encodeVal(node: nodes.ValNode): Uint8Array { const writer = this.enc.writer; const child = node.node(); writer.reset(); @@ -95,7 +96,7 @@ export class Encoder { return writer.flush(); } - public encodeObj(node: ObjNode): Uint8Array { + public encodeObj(node: nodes.ObjNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; writer.reset(); @@ -110,7 +111,23 @@ export class Encoder { this.ts(value); }; - public encodeStr(node: StrNode): Uint8Array { + public encodeVec(node: nodes.VecNode): Uint8Array { + const writer = this.enc.writer; + writer.reset(); + const length = node.elements.length; + this.writeTL(CRDT_MAJOR_OVERLAY.VEC, length); + for (let i = 0; i < length; i++) { + const childId = node.val(i); + if (!childId) writer.u8(0); + else { + writer.u8(1); + this.ts(childId); + } + } + return writer.flush(); + } + + public encodeStr(node: nodes.StrNode): Uint8Array { throw new Error('TODO'); // const encoder = this.enc; // const writer = encoder.writer; @@ -124,7 +141,7 @@ export class Encoder { // return writer.flush(); } - public encodeBin(node: BinNode): Uint8Array { + public encodeBin(node: nodes.BinNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; writer.reset(); @@ -140,7 +157,7 @@ export class Encoder { return writer.flush(); } - public encodeArr(node: ArrNode): Uint8Array { + public encodeArr(node: nodes.ArrNode): Uint8Array { const encoder = this.enc; const writer = encoder.writer; writer.reset(); diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index d081679260..56b591d168 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -93,6 +93,14 @@ describe.only('basic types', () => { const model = Model.withLogicalClock(); model.api.root({foo: null}); const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('vec', () => { + const model = Model.withLogicalClock(); + model.api.root(s.vec(s.con(false))); + const encoded = encoder.encode(model); console.log(model + ''); console.log(encoded); const decoded = decoder.decode(encoded); diff --git a/src/json-crdt/codec/structural/binary/Decoder.ts b/src/json-crdt/codec/structural/binary/Decoder.ts index 4d0a6e8e56..59d8713837 100644 --- a/src/json-crdt/codec/structural/binary/Decoder.ts +++ b/src/json-crdt/codec/structural/binary/Decoder.ts @@ -92,10 +92,10 @@ export class Decoder extends CborDecoderBase { return this.cCon(id, length); case CRDT_MAJOR.VAL: return this.cVal(id); - case CRDT_MAJOR.VEC: - return this.cVec(id, length); case CRDT_MAJOR.OBJ: return this.cObj(id, length); + case CRDT_MAJOR.VEC: + return this.cVec(id, length); case CRDT_MAJOR.STR: return this.cStr(id, length); case CRDT_MAJOR.BIN: diff --git a/src/json-crdt/nodes/vec/VecNode.ts b/src/json-crdt/nodes/vec/VecNode.ts index c7d1da5a66..6a6b0edaca 100644 --- a/src/json-crdt/nodes/vec/VecNode.ts +++ b/src/json-crdt/nodes/vec/VecNode.ts @@ -48,7 +48,7 @@ export class VecNode * @returns JSON CRDT node at the given index, if any. */ public get(index: Index): undefined | Value[Index] { - const id = this.val(index); + const id = this.elements[index] as ITimestampStruct | undefined; if (!id) return undefined; return this.doc.index.get(id); } From e993874ec3d6c8ebc0cffddd8749dfd784f6a1e1 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 10:38:13 +0100 Subject: [PATCH 06/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20s?= =?UTF-8?q?upport=20for=20"str"=20node=20type=20in=20JSON=20CRDT=20indexed?= =?UTF-8?q?=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 37 +++++++++++-------- src/json-crdt/codec/indexed/binary/Encoder.ts | 23 ++++++------ .../indexed/binary/__tests__/codec.spec.ts | 18 +++++++++ .../codec/structural/binary/Decoder.ts | 32 ++++++++-------- 4 files changed, 68 insertions(+), 42 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 480e06458e..06d360c381 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -76,12 +76,12 @@ export class Decoder { return this.decodeObj(id, length); case CRDT_MAJOR.VEC: return this.decodeVec(id, length); - // case CRDT_MAJOR.STR: - // return this.cStr(id, length); - // case CRDT_MAJOR.BIN: - // return this.cBin(id, length); - // case CRDT_MAJOR.ARR: - // return this.cArr(id, length); + case CRDT_MAJOR.STR: + return this.decodeStr(id, length); + case CRDT_MAJOR.BIN: + return this.cBin(id, length); + case CRDT_MAJOR.ARR: + return this.cArr(id, length); } return UNDEFINED; } @@ -123,19 +123,26 @@ export class Decoder { return node; } - protected cStr(id: ITimestampStruct, length: number): nodes.StrNode { - const decoder = this.dec; + protected decodeStr(id: ITimestampStruct, length: number): nodes.StrNode { const node = new nodes.StrNode(id); - node.ingest(length, () => { - const chunkId = this.ts(); - const val = decoder.val(); - if (typeof val === 'number') return new nodes.StrChunk(chunkId, val, ''); - const data = String(val); - return new nodes.StrChunk(chunkId, data.length, data); - }); + node.ingest(length, this.decodeStrChunk); return node; } + private decodeStrChunk = (): nodes.StrChunk => { + const decoder = this.dec; + const reader = decoder.reader; + const id = this.ts(); + const isTombstone = reader.uint8[reader.x] === 0; + if (isTombstone) { + reader.x++; + const length = reader.vu39(); + return new nodes.StrChunk(id, length, ''); + } + const text: string = decoder.readAsStr() as string; + return new nodes.StrChunk(id, text.length, text); + }; + protected cBin(id: ITimestampStruct, length: number): nodes.BinNode { const decoder = this.dec; const reader = decoder.reader; diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 6d3951eca1..c8acbb0c26 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -128,17 +128,18 @@ export class Encoder { } public encodeStr(node: nodes.StrNode): Uint8Array { - throw new Error('TODO'); - // const encoder = this.enc; - // const writer = encoder.writer; - // writer.reset(); - // encoder.writeStrHdr(node.size()); - // for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { - // this.ts(chunk.id); - // if (chunk.del) encoder.u32(chunk.span); - // else encoder.encodeString(chunk.data!); - // } - // return writer.flush(); + const encoder = this.enc; + const writer = encoder.writer; + writer.reset(); + this.writeTL(CRDT_MAJOR_OVERLAY.STR, node.count); + for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { + this.ts(chunk.id); + if (chunk.del) { + writer.u8(0); + writer.vu39(chunk.span); + } else encoder.writeStr(chunk.data!); + } + return writer.flush(); } public encodeBin(node: nodes.BinNode): Uint8Array { diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index 56b591d168..69ace3520d 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -101,6 +101,24 @@ describe.only('basic types', () => { const model = Model.withLogicalClock(); model.api.root(s.vec(s.con(false))); const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('str', () => { + const model = Model.withLogicalClock(); + model.api.root(''); + const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('str - 2', () => { + const model = Model.withLogicalClock(); + model.api.root('Hello, '); + model.api.str([]).ins(7, 'world!'); + model.api.str([]).del(5, 1); + const encoded = encoder.encode(model); console.log(model + ''); console.log(encoded); const decoded = decoder.decode(encoded); diff --git a/src/json-crdt/codec/structural/binary/Decoder.ts b/src/json-crdt/codec/structural/binary/Decoder.ts index 59d8713837..ec2867e150 100644 --- a/src/json-crdt/codec/structural/binary/Decoder.ts +++ b/src/json-crdt/codec/structural/binary/Decoder.ts @@ -149,22 +149,6 @@ export class Decoder extends CborDecoderBase { return obj; } - protected cArr(id: ITimestampStruct, length: number): ArrNode { - const obj = new ArrNode(this.doc, id); - obj.ingest(length, this.cArrChunk); - this.doc.index.set(id, obj); - return obj; - } - - private readonly cArrChunk = (): ArrChunk => { - const [deleted, length] = this.reader.b1vu28(); - const id = this.ts(); - if (deleted) return new ArrChunk(id, length, undefined); - const ids: ITimestampStruct[] = []; - for (let i = 0; i < length; i++) ids.push(this.cNode().id); - return new ArrChunk(id, length, ids); - }; - protected cStr(id: ITimestampStruct, length: number): StrNode { const node = new StrNode(id); if (length) node.ingest(length, this.cStrChunk); @@ -199,4 +183,20 @@ export class Decoder extends CborDecoderBase { if (deleted) return new BinChunk(id, length, undefined); else return new BinChunk(id, length, reader.buf(length)); }; + + protected cArr(id: ITimestampStruct, length: number): ArrNode { + const obj = new ArrNode(this.doc, id); + obj.ingest(length, this.cArrChunk); + this.doc.index.set(id, obj); + return obj; + } + + private readonly cArrChunk = (): ArrChunk => { + const [deleted, length] = this.reader.b1vu28(); + const id = this.ts(); + if (deleted) return new ArrChunk(id, length, undefined); + const ids: ITimestampStruct[] = []; + for (let i = 0; i < length; i++) ids.push(this.cNode().id); + return new ArrChunk(id, length, ids); + }; } From ec1c309b3fe345d0e8197aae757776b0767c979b Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 10:55:54 +0100 Subject: [PATCH 07/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20s?= =?UTF-8?q?upport=20for=20"bin"=20node=20type=20in=20JSON=20CRDT=20indexed?= =?UTF-8?q?=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 24 +++++++++---------- src/json-crdt/codec/indexed/binary/Encoder.ts | 8 +++---- .../indexed/binary/__tests__/codec.spec.ts | 18 ++++++++++++++ .../codec/structural/binary/Decoder.ts | 2 +- .../codec/structural/binary/Encoder.ts | 4 ++++ 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 06d360c381..20bd9d998b 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -79,7 +79,7 @@ export class Decoder { case CRDT_MAJOR.STR: return this.decodeStr(id, length); case CRDT_MAJOR.BIN: - return this.cBin(id, length); + return this.decodeBin(id, length); case CRDT_MAJOR.ARR: return this.cArr(id, length); } @@ -125,7 +125,7 @@ export class Decoder { protected decodeStr(id: ITimestampStruct, length: number): nodes.StrNode { const node = new nodes.StrNode(id); - node.ingest(length, this.decodeStrChunk); + if (length) node.ingest(length, this.decodeStrChunk); return node; } @@ -143,20 +143,20 @@ export class Decoder { return new nodes.StrChunk(id, text.length, text); }; - protected cBin(id: ITimestampStruct, length: number): nodes.BinNode { - const decoder = this.dec; - const reader = decoder.reader; + protected decodeBin(id: ITimestampStruct, length: number): nodes.BinNode { const node = new nodes.BinNode(id); - node.ingest(length, () => { - const chunkId = this.ts(); - const [deleted, length] = reader.b1vu28(); - if (deleted) return new nodes.BinChunk(chunkId, length, undefined); - const data = reader.buf(length); - return new nodes.BinChunk(chunkId, length, data); - }); + if (length) node.ingest(length, this.decodeBinChunk); return node; } + private decodeBinChunk = (): nodes.BinChunk => { + const id = this.ts(); + const reader = this.dec.reader; + const [deleted, length] = reader.b1vu56(); + if (deleted) return new nodes.BinChunk(id, length, undefined); + else return new nodes.BinChunk(id, length, reader.buf(length)); + }; + protected cArr(id: ITimestampStruct, length: number): nodes.ArrNode { const decoder = this.dec; const reader = decoder.reader; diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index c8acbb0c26..2468ec3d84 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -146,12 +146,12 @@ export class Encoder { const encoder = this.enc; const writer = encoder.writer; writer.reset(); - encoder.writeBinHdr(node.size()); + this.writeTL(CRDT_MAJOR_OVERLAY.BIN, node.count); for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { - this.ts(chunk.id); - const deleted = chunk.del; const length = chunk.span; - writer.b1vu28(deleted, length); + const deleted = chunk.del; + this.ts(chunk.id); + writer.b1vu56(~~deleted as 0 | 1, length); if (deleted) continue; writer.buf(chunk.data!, length); } diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index 69ace3520d..9f51f39a4b 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -119,6 +119,24 @@ describe.only('basic types', () => { model.api.str([]).ins(7, 'world!'); model.api.str([]).del(5, 1); const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('bin', () => { + const model = Model.withLogicalClock(); + model.api.root(new Uint8Array([])); + const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('bin - 2', () => { + const model = Model.withLogicalClock(); + model.api.root(new Uint8Array([1])); + model.api.bin([]).ins(1, new Uint8Array([2, 3, 4])); + model.api.bin([]).del(2, 1); + const encoded = encoder.encode(model); console.log(model + ''); console.log(encoded); const decoded = decoder.decode(encoded); diff --git a/src/json-crdt/codec/structural/binary/Decoder.ts b/src/json-crdt/codec/structural/binary/Decoder.ts index ec2867e150..c45cc98ab6 100644 --- a/src/json-crdt/codec/structural/binary/Decoder.ts +++ b/src/json-crdt/codec/structural/binary/Decoder.ts @@ -186,7 +186,7 @@ export class Decoder extends CborDecoderBase { protected cArr(id: ITimestampStruct, length: number): ArrNode { const obj = new ArrNode(this.doc, id); - obj.ingest(length, this.cArrChunk); + if (length) obj.ingest(length, this.cArrChunk); this.doc.index.set(id, obj); return obj; } diff --git a/src/json-crdt/codec/structural/binary/Encoder.ts b/src/json-crdt/codec/structural/binary/Encoder.ts index 596c22ddb1..9a26074521 100644 --- a/src/json-crdt/codec/structural/binary/Encoder.ts +++ b/src/json-crdt/codec/structural/binary/Encoder.ts @@ -159,6 +159,8 @@ export class Encoder extends CborEncoder { ts(node.id); this.writeTL(CRDT_MAJOR_OVERLAY.BIN, node.count); for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { + // TODO: Encode ID first + // TODO: Use b1vu56 const length = chunk.span; const deleted = chunk.del; writer.b1vu28(chunk.del, length); @@ -175,6 +177,8 @@ export class Encoder extends CborEncoder { this.writeTL(CRDT_MAJOR_OVERLAY.ARR, node.count); const index = this.doc.index; for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { + // TODO: Encode ID first + // TODO: Use b1vu56 const span = chunk.span; const deleted = chunk.del; writer.b1vu28(deleted, span); From b925cc3b8d448109704cf3ca61b4d47352dad8a9 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:05:11 +0100 Subject: [PATCH 08/13] =?UTF-8?q?feat(json-crdt):=20=F0=9F=8E=B8=20add=20s?= =?UTF-8?q?upport=20for=20"arr"=20node=20type=20in=20JSON=20CRDT=20indexed?= =?UTF-8?q?=20codec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 31 ++++++++++--------- src/json-crdt/codec/indexed/binary/Encoder.ts | 8 ++--- .../indexed/binary/__tests__/codec.spec.ts | 20 ++++++++++-- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 20bd9d998b..74868953ca 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -81,7 +81,7 @@ export class Decoder { case CRDT_MAJOR.BIN: return this.decodeBin(id, length); case CRDT_MAJOR.ARR: - return this.cArr(id, length); + return this.decodeArr(id, length); } return UNDEFINED; } @@ -125,7 +125,7 @@ export class Decoder { protected decodeStr(id: ITimestampStruct, length: number): nodes.StrNode { const node = new nodes.StrNode(id); - if (length) node.ingest(length, this.decodeStrChunk); + node.ingest(length, this.decodeStrChunk); return node; } @@ -145,7 +145,7 @@ export class Decoder { protected decodeBin(id: ITimestampStruct, length: number): nodes.BinNode { const node = new nodes.BinNode(id); - if (length) node.ingest(length, this.decodeBinChunk); + node.ingest(length, this.decodeBinChunk); return node; } @@ -157,18 +157,21 @@ export class Decoder { else return new nodes.BinChunk(id, length, reader.buf(length)); }; - protected cArr(id: ITimestampStruct, length: number): nodes.ArrNode { - const decoder = this.dec; - const reader = decoder.reader; + protected decodeArr(id: ITimestampStruct, length: number): nodes.ArrNode { const node = new nodes.ArrNode(this.doc, id); - node.ingest(length, () => { - const chunkId = this.ts(); - const [deleted, length] = reader.b1vu28(); - if (deleted) return new nodes.ArrChunk(chunkId, length, undefined); - const data: ITimestampStruct[] = []; - for (let i = 0; i < length; i++) data.push(this.ts()); - return new nodes.ArrChunk(chunkId, length, data); - }); + node.ingest(length, this.decodeArrChunk); return node; } + + private decodeArrChunk = (): nodes.ArrChunk => { + const id = this.ts(); + const reader = this.dec.reader; + const [deleted, length] = reader.b1vu56(); + if (deleted) return new nodes.ArrChunk(id, length, undefined); + else { + const data: ITimestampStruct[] = []; + for (let i = 0; i < length; i++) data.push(this.ts()); + return new nodes.ArrChunk(id, length, data); + } + }; } diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 2468ec3d84..249cc8220b 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -162,15 +162,15 @@ export class Encoder { const encoder = this.enc; const writer = encoder.writer; writer.reset(); - encoder.writeArrHdr(node.size()); + this.writeTL(CRDT_MAJOR_OVERLAY.ARR, node.count); for (let chunk = node.first(); chunk; chunk = node.next(chunk)) { const length = chunk.span; const deleted = chunk.del; this.ts(chunk.id); - writer.b1vu28(deleted, length); + writer.b1vu56(~~deleted as 0 | 1, length); if (deleted) continue; - const data = chunk.data!; - for (let i = 0; i < length; i++) this.ts(data[i]); + const data = chunk.data; + for (let i = 0; i < length; i++) this.ts(data![i]); } return writer.flush(); } diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index 9f51f39a4b..f56cd28504 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -137,8 +137,24 @@ describe.only('basic types', () => { model.api.bin([]).ins(1, new Uint8Array([2, 3, 4])); model.api.bin([]).del(2, 1); const encoded = encoder.encode(model); - console.log(model + ''); - console.log(encoded); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('arr', () => { + const model = Model.withLogicalClock(); + model.api.root([-1]); + const encoded = encoder.encode(model); + const decoded = decoder.decode(encoded); + expect(decoded.view()).toStrictEqual(model.view()); + }); + + test('arr - 2', () => { + const model = Model.withLogicalClock(); + model.api.root([-1]); + model.api.arr([]).ins(1, [2, 3, 4]); + model.api.arr([]).del(2, 1); + const encoded = encoder.encode(model); const decoded = decoder.decode(encoded); expect(decoded.view()).toStrictEqual(model.view()); }); From 4391c3c903e4b3f65a21260af7ec4fcf865a7886 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:05:33 +0100 Subject: [PATCH 09/13] =?UTF-8?q?style(json-crdt):=20=F0=9F=92=84=20run=20?= =?UTF-8?q?Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Decoder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json-crdt/codec/indexed/binary/Decoder.ts b/src/json-crdt/codec/indexed/binary/Decoder.ts index 74868953ca..899ceb4256 100644 --- a/src/json-crdt/codec/indexed/binary/Decoder.ts +++ b/src/json-crdt/codec/indexed/binary/Decoder.ts @@ -110,7 +110,7 @@ export class Decoder { } return obj; } - + public decodeVec(id: ITimestampStruct, length: number): nodes.VecNode { const reader = this.dec.reader; const node = new nodes.VecNode(this.doc, id); From 58069f6246f2fd0e64952dd8b0fe6089bb51e3eb Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:06:29 +0100 Subject: [PATCH 10/13] =?UTF-8?q?test(json-crdt):=20=F0=9F=92=8D=20enable?= =?UTF-8?q?=20skippe=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts index f56cd28504..2d5d8632ce 100644 --- a/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts +++ b/src/json-crdt/codec/indexed/binary/__tests__/codec.spec.ts @@ -72,7 +72,7 @@ test('can encode ID as const value', () => { expect(equal(ts, new Timestamp(model.clock.sid, 2))).toBe(true); }); -describe.only('basic types', () => { +describe('basic types', () => { test('con', () => { const model = Model.withLogicalClock(); model.api.root(konst(123)); From 1f431684508448a0563de373fc65d72918680424 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:29:43 +0100 Subject: [PATCH 11/13] =?UTF-8?q?refactor(json-crdt):=20=F0=9F=92=A1=20rem?= =?UTF-8?q?ove=20unnecessary=20variables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/Encoder.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/json-crdt/codec/indexed/binary/Encoder.ts b/src/json-crdt/codec/indexed/binary/Encoder.ts index 249cc8220b..b29cef09a8 100644 --- a/src/json-crdt/codec/indexed/binary/Encoder.ts +++ b/src/json-crdt/codec/indexed/binary/Encoder.ts @@ -7,12 +7,9 @@ import * as nodes from '../../../nodes'; import {CRDT_MAJOR_OVERLAY} from '../../structural/binary/constants'; import {IndexedFields, FieldName} from './types'; -const EMPTY = new Uint8Array(0); - export class Encoder { public readonly enc: CborEncoder; protected clockTable?: ClockTable; - protected model?: IndexedFields; constructor(writer?: CrdtWriter) { this.enc = new CborEncoder(writer || new CrdtWriter()); @@ -25,27 +22,25 @@ export class Encoder { clockTable.write(writer); const encodedClock = writer.flush(); const rootValueId = doc.root.val; - const model: IndexedFields = (this.model = { + const result: IndexedFields = { c: encodedClock, - }); + }; if (rootValueId.sid !== 0) { writer.reset(); this.ts(rootValueId); - model.r = writer.flush(); + result.r = writer.flush(); } - doc.index.forEach(({v: node}) => this.onNode(node)); - return model; + doc.index.forEach(({v: node}) => this.onNode(result, node)); + return result; } - protected readonly onNode = (node: nodes.JsonNode) => { + protected readonly onNode = (result: IndexedFields, node: nodes.JsonNode) => { const id = node.id; const sid = id.sid; const time = id.time; - const model = this.model!; const sidIndex = this.clockTable!.getBySid(sid).index; - const sidFieldPart = sidIndex.toString(36) + '_'; - const field = (sidFieldPart + time.toString(36)) as FieldName; - model[field] = this.encodeNode(node); + const field = (sidIndex.toString(36) + '_' + time.toString(36)) as FieldName; + result[field] = this.encodeNode(node); }; public encodeNode(node: nodes.JsonNode): Uint8Array { @@ -56,7 +51,7 @@ export class Encoder { else if (node instanceof nodes.StrNode) return this.encodeStr(node); else if (node instanceof nodes.BinNode) return this.encodeBin(node); else if (node instanceof nodes.ArrNode) return this.encodeArr(node); - else return EMPTY; + throw new Error('UNKNOWN_NODE'); } protected ts(id: ITimestampStruct): void { From 11a4934064cd9cea3149076c8276cf7402138771 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:39:59 +0100 Subject: [PATCH 12/13] =?UTF-8?q?docs(json-crdt):=20=E2=9C=8F=EF=B8=8F=20a?= =?UTF-8?q?dd=20indexed=20codec=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/codec/indexed/binary/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/json-crdt/codec/indexed/binary/README.md diff --git a/src/json-crdt/codec/indexed/binary/README.md b/src/json-crdt/codec/indexed/binary/README.md new file mode 100644 index 0000000000..b982047a30 --- /dev/null +++ b/src/json-crdt/codec/indexed/binary/README.md @@ -0,0 +1,13 @@ +## Binary Indexed Format + +The Binary Index encoding format for JSON CRDT returns a flat map, where each +key is a string and each value is an `Uint8Array` blob. + +The map has the following keys: + +- `"r"` - ID of the root node. +- `"c"` - Clock table of the document. +- `"{sid}_{time}` - Each key is a string of the form `{sid}_{time}`, where + `sid` is the session ID index in the clock table encoded as Base36 and `time` + is the logical clock sequence number encoded as Base36. The value is the + encoded node. From 40c43ba0c4ce01475a36328abd821aea59778629 Mon Sep 17 00:00:00 2001 From: streamich Date: Mon, 6 Nov 2023 11:44:16 +0100 Subject: [PATCH 13/13] =?UTF-8?q?test(json-crdt):=20=F0=9F=92=8D=20add=20i?= =?UTF-8?q?ndexed=20JSON=20CRDT=20codec=20to=20fuzzer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/json-crdt/__tests__/fuzzer/SessionLogical.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/json-crdt/__tests__/fuzzer/SessionLogical.ts b/src/json-crdt/__tests__/fuzzer/SessionLogical.ts index e454ca5280..21cac3cf0e 100644 --- a/src/json-crdt/__tests__/fuzzer/SessionLogical.ts +++ b/src/json-crdt/__tests__/fuzzer/SessionLogical.ts @@ -10,6 +10,8 @@ import {encode as encodeJson} from '../../../json-crdt-patch/codec/verbose/encod import {Encoder as BinaryEncoder} from '../../codec/structural/binary/Encoder'; import {Encoder as CompactEncoder} from '../../codec/structural/compact/Encoder'; import {Encoder as JsonEncoder} from '../../codec/structural/verbose/Encoder'; +import {Encoder as IndexedBinaryEncoder} from '../../codec/indexed/binary/Encoder'; +import {Decoder as IndexedBinaryDecoder} from '../../codec/indexed/binary/Decoder'; import {generateInteger} from './util'; import {Model} from '../..'; import {Patch} from '../../../json-crdt-patch/Patch'; @@ -27,6 +29,8 @@ const compactEncoder = new CompactEncoder(); const compactDecoder = new CompactDecoder(); const binaryEncoder = new BinaryEncoder(); const binaryDecoder = new BinaryDecoder(); +const indexedBinaryEncoder = new IndexedBinaryEncoder(); +const indexedBinaryDecoder = new IndexedBinaryDecoder(); export class SessionLogical { public models: Model[] = []; @@ -192,6 +196,7 @@ export class SessionLogical { if (randomU32(0, 1)) model = jsonDecoder.decode(jsonEncoder.encode(model)); if (randomU32(0, 1)) model = compactDecoder.decode(compactEncoder.encode(model)); if (randomU32(0, 1)) model = binaryDecoder.decode(binaryEncoder.encode(model)); + if (randomU32(0, 1)) model = indexedBinaryDecoder.decode(indexedBinaryEncoder.encode(model)); } for (let j = 0; j < this.concurrency; j++) { const patches = this.patches[j];