feat: use msgpack (#450)

Signed-off-by: Sacha Froment <[email protected]> Co-authored-by: hoangquocvietuet <[email protected]> Co-authored-by: droak <[email protected]>
drp-tech · Feb 12, 2025 · 64ff132 · 64ff132
1 parent e72e81b
commit 64ff132
Show file tree

Hide file tree

Showing 7 changed files with 428 additions and 143 deletions.
diff --git a/eslint.config.mjs b/eslint.config.mjs
@@ -27,7 +27,6 @@ const config = tseslint.config(
 			"**/*.log",
 			"**/*_pb.js",
 			"**/*_pb.ts",
-			"**/serializer.ts",
 		],
 	},
 	eslint.configs.recommended,

diff --git a/packages/node/tests/utils.test.ts b/packages/node/tests/utils.test.ts
@@ -0,0 +1,27 @@
+import { SetDRP } from "@ts-drp/blueprints/src/index.js";
+import { DRPObject } from "@ts-drp/object/src/index.js";
+import { beforeAll, describe, expect, test } from "vitest";
+
+import { deserializeStateMessage, serializeStateMessage } from "../src/utils.js";
+
+describe("State message utils", () => {
+	let object: DRPObject;
+
+	beforeAll(async () => {
+		object = DRPObject.createObject({
+			peerId: "test",
+			id: "test",
+			drp: new SetDRP<number>(),
+		});
+		(object.drp as SetDRP<number>).add(1);
+		(object.drp as SetDRP<number>).add(2);
+		(object.drp as SetDRP<number>).add(3);
+	});
+
+	test("Should serialize/deserialize state message", async () => {
+		const state = object["_computeDRPState"].bind(object);
+		const serialized = serializeStateMessage(state(object.hashGraph.getFrontier()));
+		const deserialized = deserializeStateMessage(serialized);
+		expect(deserialized).toStrictEqual(state(object.hashGraph.getFrontier()));
+	});
+});
diff --git a/packages/object/package.json b/packages/object/package.json
@@ -37,6 +37,7 @@
  },
  "dependencies": {
   "@chainsafe/bls": "^8.1.0",
+  "@msgpack/msgpack": "^3.0.1",
   "@ts-drp/logger": "^0.7.0",
   "es-toolkit": "1.30.1",
   "fast-deep-equal": "^3.1.3",

diff --git a/packages/object/src/utils/serializer.ts b/packages/object/src/utils/serializer.ts
@@ -1,141 +1,68 @@
-import { Value } from "../proto/google/protobuf/struct_pb.js";
-
-export function serializeValue(obj: any): Uint8Array {
-	const serialized = _serializeToJSON(obj);
-	return Value.encode(Value.wrap(serialized)).finish();
-}
-
-export function deserializeValue(value: any): any {
-	const bytes = new Uint8Array(_objectValues(value));
-	const v = Value.decode(bytes);
-	const unwrapped = Value.unwrap(v);
-	return _deserializeFromJSON(unwrapped);
-}
-
-function _objectValues(obj: any): any[] {
-	const tmp: any[] = [];
-	for (const key in obj) {
-		tmp.push(obj[key]);
-	}
-	return tmp;
-}
-
-function _serializeToJSON(obj: any): any {
-	// Handle null/undefined
-	if (obj == null) return null;
-
-	// Handle primitive types
-	if (typeof obj !== "object") return obj;
-
-	// Handle Date objects
-	if (obj instanceof Date) {
-		return {
-			__type: "Date",
-			value: obj.toISOString(),
-		};
-	}
-
-	// Handle Maps
-	if (obj instanceof Map) {
-		return {
-			__type: "Map",
-			value: Array.from(obj.entries()),
-		};
-	}
-
-	// Handle Sets
-	if (obj instanceof Set) {
-		return {
-			__type: "Set",
-			value: Array.from(obj.values()),
-		};
-	}
-
-	// Handle regular arrays
-	if (Array.isArray(obj)) {
-		return obj.map((item) => _serializeToJSON(item));
-	}
-
-	// Handle regular objects
-	const result: any = {};
-	for (const [key, value] of Object.entries(obj)) {
-		// Skip non-enumerable properties and functions
-		if (typeof value === "function") continue;
-
-		// Handle circular references
-		try {
-			result[key] = _serializeToJSON(value);
-		} catch (e) {
-			console.warn(`Circular reference detected for key: ${key}`);
-			result[key] = null;
+import { encode, decode, ExtensionCodec } from "@msgpack/msgpack";
+
+const extensionCodec = new ExtensionCodec();
+
+const SET_EXT_TYPE = 0; // Any in 0-127
+extensionCodec.register({
+	type: SET_EXT_TYPE,
+	encode: (object: unknown): Uint8Array | null => {
+		if (object instanceof Set) {
+			return encode([...object], { extensionCodec });
+		} else {
+			return null;
 		}
-	}
-
-	// Add class name if available
-	if (obj.constructor && obj.constructor.name !== "Object") {
-		result.__type = obj.constructor.name;
-	}
-
-	return result;
-}
-
-function _deserializeFromJSON(obj: any): any {
-	// Handle null/undefined
-	if (obj == null) return obj;
-
-	// Handle primitive types
-	if (typeof obj !== "object") return obj;
-
-	// Handle arrays
-	if (Array.isArray(obj)) {
-		return obj.map((item) => _deserializeFromJSON(item));
-	}
-
-	// Handle special types
-	if (obj.__type) {
-		switch (obj.__type) {
-			case "Date":
-				return new Date(obj.value);
-
-			case "Map":
-				return new Map(
-					obj.value.map(([k, v]: [any, any]) => [_deserializeFromJSON(k), _deserializeFromJSON(v)])
-				);
-
-			case "Set":
-				return new Set(obj.value.map((v: any) => _deserializeFromJSON(v)));
-
-			case "Uint8Array":
-				return new Uint8Array(obj.value);
-
-			case "Float32Array":
-				return new Float32Array(obj.value);
-
-			// Add other TypedArrays as needed
-
-			default:
-				// Try to reconstruct custom class if available
-				try {
-					const CustomClass = globalThis[obj.__type as keyof typeof globalThis];
-					if (typeof CustomClass === "function") {
-						return Object.assign(
-							new CustomClass(),
-							_deserializeFromJSON({ ...obj, __type: undefined })
-						);
-					}
-				} catch (e) {
-					console.warn(`Could not reconstruct class ${obj.__type}`);
-				}
+	},
+	decode: (data: Uint8Array) => {
+		const array = decode(data, { extensionCodec }) as Array<unknown>;
+		return new Set(array);
+	},
+});
+
+// Map<K, V>
+const MAP_EXT_TYPE = 1; // Any in 0-127
+extensionCodec.register({
+	type: MAP_EXT_TYPE,
+	encode: (object: unknown): Uint8Array | null => {
+		if (object instanceof Map) {
+			return encode([...object], { extensionCodec });
+		} else {
+			return null;
 		}
-	}
-
-	// Handle regular objects
-	const result: any = {};
-	for (const [key, value] of Object.entries(obj)) {
-		if (key !== "__type") {
-			result[key] = _deserializeFromJSON(value);
+	},
+	decode: (data: Uint8Array) => {
+		const array = decode(data, { extensionCodec }) as Array<[unknown, unknown]>;
+		return new Map(array);
+	},
+});
+
+const FLOAT_32_ARRAY_EXT_TYPE = 2; // Any in 0-127
+extensionCodec.register({
+	type: FLOAT_32_ARRAY_EXT_TYPE,
+	encode: (object: unknown): Uint8Array | null => {
+		if (object instanceof Float32Array) {
+			return encode([...object], { extensionCodec });
+		} else {
+			return null;
 		}
-	}
+	},
+	decode: (data: Uint8Array) => {
+		const array = decode(data, { extensionCodec }) as Array<number>;
+		return new Float32Array(array);
+	},
+});
+
+/**
+ * Main entry point for serialization.
+ * Converts any value into a Uint8Array using Protocol Buffers.
+ */
+export function serializeValue(obj: unknown): Uint8Array {
+	return encode(obj, { extensionCodec });
+}
 
-	return result;
+/**
+ * Main entry point for deserialization.
+ * Converts a Uint8Array back into the original value structure.
+ */
+export function deserializeValue(value: Uint8Array): unknown {
+	return decode(value, { extensionCodec });
 }
diff --git a/packages/object/tests/serializer.bench.ts b/packages/object/tests/serializer.bench.ts
@@ -0,0 +1,110 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import Benchmark from "benchmark";
+
+import { deserializeValue } from "../dist/src/index.js";
+import { serializeValue } from "../src/utils/serializer.js";
+function createNestedObject(depth: number, breadth: number): any {
+	if (depth <= 0) {
+		return {
+			num: Math.random(),
+			str: "test",
+			date: new Date(),
+			set: new Set([1, 2, 3]),
+			map: new Map([
+				["a", 1],
+				["b", 2],
+			]),
+			array: new Uint8Array([1, 2, 3, 4]),
+			float: new Float32Array([1.1, 2.2, 3.3]),
+		};
+	}
+	const obj: any = {};
+	for (let i = 0; i < breadth; i++) {
+		obj[`child${i}`] = createNestedObject(depth - 1, breadth);
+	}
+	return obj;
+}
+
+const suite = new Benchmark.Suite();
+function benchmarkSerializeValue(depth: number, breadth: number) {
+	return suite.add(`Serialize ${depth} depth ${breadth} breadth`, () => {
+		// Create a deeply nested structure
+		// Create test data with depth=5 and breadth=3
+		// This creates 3^5 = 243 leaf nodes, each with 7 complex properties
+		const deepObject = createNestedObject(depth, breadth);
+		// Warm up
+		for (let i = 0; i < 3; i++) {
+			serializeValue(deepObject);
+		}
+		// Benchmark
+		const iterations = 100;
+		const start = performance.now();
+		for (let i = 0; i < iterations; i++) {
+			serializeValue(deepObject);
+		}
+		const end = performance.now();
+		const avgMs = (end - start) / iterations;
+		const leaf = Math.pow(depth, breadth);
+		console.log(`Average serialization time: ${avgMs.toFixed(2)}ms`);
+		console.log(`Object stats:
+			- Depth: ${depth}
+			- Breadth: ${breadth}
+			- Leaf nodes: ${leaf}
+			- Complex properties per leaf: 7
+			- Total complex values: ${leaf * 7}
+	 `);
+	});
+}
+
+benchmarkSerializeValue(5, 5);
+
+suite
+	.on("cycle", (event: Benchmark.Event) => {
+		console.log(String(event.target));
+	})
+	.on("complete", function (this: Benchmark.Suite) {
+		console.log(`Fastest is ${this.filter("fastest").map("name")}`);
+	})
+	.run({ async: true });
+
+function benchmarkDeserializeValue(depth: number, breadth: number) {
+	return suite.add(`Deserialize ${depth} depth ${breadth} breadth`, () => {
+		// Create a deeply nested structure
+		// Create test data with depth=5 and breadth=3
+		// This creates 3^5 = 243 leaf nodes, each with 7 complex properties
+		const deepObject = createNestedObject(depth, breadth);
+		const serialized = serializeValue(deepObject);
+		// Warm up
+		for (let i = 0; i < 3; i++) {
+			deserializeValue(serialized);
+		}
+		// Benchmark
+		const iterations = 100;
+		const start = performance.now();
+		for (let i = 0; i < iterations; i++) {
+			deserializeValue(serialized);
+		}
+		const end = performance.now();
+		const avgMs = (end - start) / iterations;
+		const leaf = Math.pow(depth, breadth);
+		console.log(`Average deserialization time: ${avgMs.toFixed(2)}ms`);
+		console.log(`Object stats:
+			- Depth: ${depth}
+			- Breadth: ${breadth}
+			- Leaf nodes: ${leaf}
+			- Complex properties per leaf: 7
+			- Total complex values: ${leaf * 7}
+	 `);
+	});
+}
+
+benchmarkDeserializeValue(5, 5);
+
+suite
+	.on("cycle", (event: Benchmark.Event) => {
+		console.log(String(event.target));
+	})
+	.on("complete", function (this: Benchmark.Suite) {
+		console.log(`Fastest is ${this.filter("fastest").map("name")}`);
+	})
+	.run({ async: true });