From 39ff6491569979d113b3a381cda99c132fb57371 Mon Sep 17 00:00:00 2001 From: Nguyen Viet Dung <29406816+magnified103@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:18:00 +0700 Subject: [PATCH] prune redundant files --- packages/node/src/riblt/decoder.ts | 113 ----------------------- packages/node/src/riblt/encoder.ts | 142 ----------------------------- packages/node/src/riblt/index.ts | 0 packages/node/src/riblt/mapping.ts | 41 --------- packages/node/src/riblt/symbol.ts | 87 ------------------ packages/node/tests/riblt.test.ts | 130 -------------------------- 6 files changed, 513 deletions(-) delete mode 100644 packages/node/src/riblt/decoder.ts delete mode 100644 packages/node/src/riblt/encoder.ts delete mode 100644 packages/node/src/riblt/index.ts delete mode 100644 packages/node/src/riblt/mapping.ts delete mode 100644 packages/node/src/riblt/symbol.ts delete mode 100644 packages/node/tests/riblt.test.ts diff --git a/packages/node/src/riblt/decoder.ts b/packages/node/src/riblt/decoder.ts deleted file mode 100644 index 76e6cb502..000000000 --- a/packages/node/src/riblt/decoder.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { CodingPrefix } from "./encoder.js"; -import type { - CodedSymbol, - HashedSymbol, - SourceSymbol, - SymbolFactory, -} from "./symbol.js"; - -export class Decoder extends CodingPrefix { - decodedLocalSymbols: T[]; - decodedRemoteSymbols: T[]; - isDecoded: boolean[]; - modifiedCodedSymbols: number[]; - visited: boolean[]; - remaining: number; - pureSymbols: CodedSymbol[]; - - constructor(sourceSymbolFactory: SymbolFactory) { - super(sourceSymbolFactory); - this.decodedLocalSymbols = []; - this.decodedRemoteSymbols = []; - this.isDecoded = []; - this.modifiedCodedSymbols = []; - this.visited = []; - this.remaining = 0; - this.pureSymbols = []; - } - - extendPrefix(size: number): void { - super.extendPrefix(size); - while (this.isDecoded.length < size) { - this.isDecoded.push(false); - this.visited.push(false); - this.remaining++; - } - } - - // called at most once for each index - addCodedSymbol( - index: number, - localSymbol: CodedSymbol, - remoteSymbol: CodedSymbol, - ): void { - this.extendPrefix(index + 1); - this.codedSymbols[index].apply(localSymbol, localSymbol.count); - this.codedSymbols[index].apply(remoteSymbol, -remoteSymbol.count); - if (!this.visited[index]) { - this.visited[index] = true; - this.modifiedCodedSymbols.push(index); - } - } - - maps(index: number, hashedSymbol: HashedSymbol, direction: number): void { - if (!this.isDecoded[index]) { - this.codedSymbols[index].apply(hashedSymbol, direction); - if (!this.visited[index]) { - this.visited[index] = true; - this.modifiedCodedSymbols.push(index); - } - } - } - - tryDecode(): boolean { - this.computePrefix(this.codedSymbols.length); - while (this.modifiedCodedSymbols.length > 0) { - const candidates: number[] = []; - for (const index of this.modifiedCodedSymbols) { - if (this.isDecoded[index]) { - continue; - } - this.visited[index] = true; - const symbol = this.codedSymbols[index]; - if (symbol.isZero()) { - // The set difference is empty. We can safely mark this index as decoded. - this.isDecoded[index] = true; - this.remaining--; - } else if (symbol.isPure()) { - // Found a (potentially) pure symbol. - candidates.push(index); - } - } - // Arrays cleanup - for (const index of this.modifiedCodedSymbols) { - this.visited[index] = false; - } - this.modifiedCodedSymbols = []; - - // Process pure symbols - for (const index of candidates) { - if (this.isDecoded[index]) { - continue; - } - const symbol = this.codedSymbols[index]; - if (symbol.isZero()) { - this.isDecoded[index] = true; - this.remaining--; - } else if (symbol.isPure()) { - const decodedSymbol = this.symbolFactory.cloneSource(symbol.sum); - if (symbol.count === 1) { - this.decodedLocalSymbols.push(decodedSymbol); - } else if (symbol.count === -1) { - this.decodedRemoteSymbols.push(decodedSymbol); - } else { - throw Error(`Invalid pure symbol ${symbol}`); - } - this.addSymbol(decodedSymbol, -symbol.count); - this.computePrefix(this.codedSymbols.length); - } - } - } - return this.remaining === 0; - } -} diff --git a/packages/node/src/riblt/encoder.ts b/packages/node/src/riblt/encoder.ts deleted file mode 100644 index 8edd24285..000000000 --- a/packages/node/src/riblt/encoder.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { RandomMapping } from "./mapping.js"; -import { - type CodedSymbol, - HashedSymbol, - type SourceSymbol, - type SymbolFactory, -} from "./symbol.js"; - -class SymbolMapping { - sourceIdx: number; - codedIdx: number; - - constructor(sourceIdx: number, codedIdx: number) { - this.sourceIdx = sourceIdx; - this.codedIdx = codedIdx; - } -} - -class MappingHeap { - // Binary heap implementation - private heap: SymbolMapping[] = []; - - private fixHead(): void { - let curr = 0; - while (true) { - let child = (curr << 1) + 1; - if (child >= this.heap.length) { - break; - } - if ( - child + 1 < this.heap.length && - this.heap[child + 1].codedIdx < this.heap[child].codedIdx - ) { - child = child + 1; - } - if (this.heap[curr].codedIdx <= this.heap[child].codedIdx) { - break; - } - [this.heap[curr], this.heap[child]] = [this.heap[child], this.heap[curr]]; - curr = child; - } - } - - private fixTail(): void { - let curr = this.heap.length - 1; - while (curr > 0) { - const parent = (curr - 1) >> 1; - if (this.heap[parent].codedIdx <= this.heap[curr].codedIdx) { - break; - } - [this.heap[parent], this.heap[curr]] = [ - this.heap[curr], - this.heap[parent], - ]; - curr = parent; - } - } - - push(mapping: SymbolMapping): void { - this.heap.push(mapping); - this.fixTail(); - } - - set top(mapping: SymbolMapping) { - if (this.heap.length === 0) throw Error("Heap is empty"); - this.heap[0] = mapping; - this.fixHead(); - } - - get top(): SymbolMapping { - if (this.heap.length === 0) throw Error("Heap is empty"); - return this.heap[0]; - } - - get size(): number { - return this.heap.length; - } -} - -export class CodingPrefix { - private sourceSymbols: HashedSymbol[]; - private sourceSymbolDirections: number[]; - public codedSymbols: CodedSymbol[]; - private mapGenerators: RandomMapping[]; - private queue: MappingHeap; - - constructor(protected readonly symbolFactory: SymbolFactory) { - this.sourceSymbols = []; - this.sourceSymbolDirections = []; - this.codedSymbols = [symbolFactory.emptyCoded()]; - this.mapGenerators = []; - this.queue = new MappingHeap(); - } - - addSymbol(symbol: T, direction = 1): void { - const hashedSymbol = new HashedSymbol( - this.symbolFactory.cloneSource(symbol), - ); - const mapping = new RandomMapping(hashedSymbol.checksum, 0); - - this.sourceSymbols.push(hashedSymbol); - this.sourceSymbolDirections.push(direction); - this.mapGenerators.push(mapping); - this.queue.push( - new SymbolMapping(this.sourceSymbols.length - 1, mapping.lastIdx), - ); - } - - maps(index: number, hashedSymbol: HashedSymbol, direction: number): void { - this.codedSymbols[index].apply(hashedSymbol, direction); - } - - extendPrefix(size: number): void { - while (this.codedSymbols.length < size) { - this.codedSymbols.push(this.symbolFactory.emptyCoded()); - } - } - - computePrefix(size: number): void { - while (this.queue.size > 0 && this.queue.top.codedIdx < size) { - const mapping = this.queue.top; - while (mapping.codedIdx < size) { - const sourceIdx = mapping.sourceIdx; - const codedIdx = mapping.codedIdx; - this.maps( - codedIdx, - this.sourceSymbols[sourceIdx], - this.sourceSymbolDirections[sourceIdx], - ); - mapping.codedIdx = this.mapGenerators[sourceIdx].nextIndex(); - } - this.queue.top = mapping; - } - } -} - -export class Encoder extends CodingPrefix { - producePrefix(size: number): void { - super.extendPrefix(size); - super.computePrefix(size); - } -} diff --git a/packages/node/src/riblt/index.ts b/packages/node/src/riblt/index.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/node/src/riblt/mapping.ts b/packages/node/src/riblt/mapping.ts deleted file mode 100644 index 0504eff3d..000000000 --- a/packages/node/src/riblt/mapping.ts +++ /dev/null @@ -1,41 +0,0 @@ -import * as crypto from "node:crypto"; - -function rotl(x: bigint, k: bigint) { - return BigInt.asUintN(64, (x << k) | (x >> (64n - k))); -} - -export class RandomMapping { - private s: BigUint64Array; - lastIdx: number; - - constructor(seed: Uint8Array, lastIdx = 0) { - this.s = new BigUint64Array( - crypto.createHash("sha256").update(seed).digest().buffer, - ); - this.lastIdx = lastIdx; - } - - nextIndex(): number { - // https://prng.di.unimi.it/xoshiro256starstar.c - const result = BigInt.asUintN( - 64, - rotl(BigInt.asUintN(64, this.s[1] * 5n), 7n) * 9n, - ); - - const t = BigInt.asUintN(64, this.s[1] << 17n); - - this.s[2] ^= this.s[0]; - this.s[3] ^= this.s[1]; - this.s[1] ^= this.s[2]; - this.s[0] ^= this.s[3]; - - this.s[2] ^= t; - - this.s[3] = rotl(this.s[3], 45n); - - this.lastIdx += Math.ceil( - (this.lastIdx + 1.5) * (2 ** 32 / Math.sqrt(Number(result) + 1) - 1), - ); - return this.lastIdx; - } -} diff --git a/packages/node/src/riblt/symbol.ts b/packages/node/src/riblt/symbol.ts deleted file mode 100644 index dc15120f8..000000000 --- a/packages/node/src/riblt/symbol.ts +++ /dev/null @@ -1,87 +0,0 @@ -export interface SourceSymbol { - xor(s: SourceSymbol): void; - hash(): Uint8Array; - toString(): string; -} - -export class HashedSymbol { - sum: T; - checksum: Uint8Array; - - constructor(sum: T, checksum?: Uint8Array) { - this.sum = sum; - if (checksum === undefined) { - this.checksum = sum.hash(); - } else { - this.checksum = checksum; - } - } - - xor(s: HashedSymbol): void { - this.sum.xor(s.sum); - for (let i = 0; i < this.checksum.length; i++) { - this.checksum[i] ^= s.checksum[i]; - } - } - - isPure(): boolean { - const checksum = this.sum.hash(); - if (checksum.length !== this.checksum.length) { - throw Error("Checksum length mismatch"); - } - for (let i = 0; i < checksum.length; i++) { - if (checksum[i] !== this.checksum[i]) { - return false; - } - } - return true; - } - - toString(): string { - return `HashedSymbol(sum=${this.sum}, hash=[${this.checksum}])`; - } -} - -export class CodedSymbol extends HashedSymbol { - count: number; - - constructor(sum: T, checksum: Uint8Array, count: number) { - super(sum, checksum); - this.count = count; - } - - apply(s: HashedSymbol, direction: number) { - super.xor(s); - this.count += direction; - } - - xor(s: CodedSymbol) { - super.xor(s); - this.count -= s.count; - } - - isZero(): boolean { - if (this.count !== 0) { - return false; - } - for (let i = 0; i < this.checksum.length; i++) { - if (this.checksum[i] !== 0) { - return false; - } - } - return true; - } - - toString(): string { - return `CodedSymbol(sum=${this.sum}, hash=[${this.checksum}], ${this.count})`; - } -} - -export abstract class SymbolFactory { - abstract emptySource(): T; - abstract emptyHash(): Uint8Array; - abstract cloneSource(s: T): T; - emptyCoded(): CodedSymbol { - return new CodedSymbol(this.emptySource(), this.emptyHash(), 0); - } -} diff --git a/packages/node/tests/riblt.test.ts b/packages/node/tests/riblt.test.ts deleted file mode 100644 index 26c9aa7bc..000000000 --- a/packages/node/tests/riblt.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import * as crypto from "node:crypto"; -import { beforeEach, describe, expect, test } from "vitest"; -import { Decoder } from "../src/riblt/decoder.js"; -import { Encoder } from "../src/riblt/encoder.js"; -import { type SourceSymbol, SymbolFactory } from "../src/riblt/symbol.js"; - -class VertexSymbol implements SourceSymbol { - data: number; - - constructor(data: number) { - this.data = data; - } - - xor(s: VertexSymbol): void { - this.data ^= s.data; - } - - hash(): Uint8Array { - return new Uint8Array( - crypto - .createHash("sha1") - .update(new Uint32Array([this.data])) - .digest(), - ); - } - - equals(s: VertexSymbol): boolean { - return this.data === s.data; - } - - toString(): string { - return `${this.data}`; - } -} - -class VertexSymbolFactory extends SymbolFactory { - emptySource(): VertexSymbol { - return new VertexSymbol(0); - } - - emptyHash(): Uint8Array { - return new Uint8Array(20); - } - - cloneSource(s: VertexSymbol): VertexSymbol { - return new VertexSymbol(s.data); - } - - newTestSymbol(i): VertexSymbol { - return new VertexSymbol(i); - } -} - -describe("RIBLT test", async () => { - const factory = new VertexSymbolFactory(); - - test.each([10, 20, 40, 100, 1000, 10000, 50000, 100000, 200000])( - "d=%i", - async (d) => { - const nlocal = d >> 1; - const nremote = d >> 1; - const ncommon = d; - - let symbolIndex = 0; - - const localEncoder = new Encoder(factory); - const remoteEncoder = new Encoder(factory); - const localDecoder = new Decoder(factory); - - enum SymbolState { - Local = 0, - Remote = 1, - Common = 2, - }; - const symbolState: SymbolState[] = []; - - for (let i = 0; i < nlocal; i++) { - const localSymbol = factory.newTestSymbol(symbolIndex); - symbolState.push(SymbolState.Local); - localEncoder.addSymbol(localSymbol); - symbolIndex++; - } - for (let i = 0; i < nremote; i++) { - const remoteSymbol = factory.newTestSymbol(symbolIndex); - symbolState.push(SymbolState.Remote); - remoteEncoder.addSymbol(remoteSymbol); - symbolIndex++; - } - for (let i = 0; i < ncommon; i++) { - const localSymbol = factory.newTestSymbol(symbolIndex); - const remoteSymbol = factory.cloneSource(localSymbol); - symbolState.push(SymbolState.Common); - localEncoder.addSymbol(localSymbol); - remoteEncoder.addSymbol(remoteSymbol); - symbolIndex++; - } - - let sequenceSize = 0; - do { - sequenceSize++; - localEncoder.producePrefix(sequenceSize); - remoteEncoder.producePrefix(sequenceSize); - localDecoder.addCodedSymbol( - sequenceSize - 1, - localEncoder.codedSymbols[sequenceSize - 1], - remoteEncoder.codedSymbols[sequenceSize - 1], - ); - } while (!localDecoder.tryDecode()); - - const visited = new Array(symbolIndex).fill(false); - - for (const symbol of localDecoder.decodedLocalSymbols) { - expect(Number.isInteger(symbol.data)).toBe(true); - expect(symbol.data >= 0 && symbol.data < symbolIndex).toBe(true); - expect(visited[symbol.data]).toBe(false); - visited[symbol.data] = true; - expect(symbolState[symbol.data]).toBe(SymbolState.Local); - } - for (const symbol of localDecoder.decodedRemoteSymbols) { - expect(Number.isInteger(symbol.data)).toBe(true); - expect(symbol.data >= 0 && symbol.data < symbolIndex).toBe(true); - expect(visited[symbol.data]).toBe(false); - visited[symbol.data] = true; - expect(symbolState[symbol.data]).toBe(SymbolState.Remote); - } - - console.log(`${sequenceSize} symbols, ${(sequenceSize / d).toFixed(3)} symbols/diff`); - }, - ); -});