diff --git a/src/helpers/codec.test.ts b/src/helpers/codec.test.ts index a7b10c4..ae7aa71 100644 --- a/src/helpers/codec.test.ts +++ b/src/helpers/codec.test.ts @@ -1,11 +1,11 @@ import { strict as assert } from "assert" -import * as _ from "lodash" import { describe, it } from "mocha" import { Tuple } from "../storage/types" import { sortedValues } from "../test/fixtures" import { decodeTuple, decodeValue, encodeTuple, encodeValue } from "./codec" import { compare } from "./compare" import { TupleToString, ValueToString } from "./compareTuple" +import { randomInt } from "./randomInt" describe("codec", () => { describe("encodeValue", () => { @@ -89,9 +89,11 @@ describe("codec", () => { const test = (aTuple: Tuple, bTuple: Tuple, result: number) => { const a = encodeTuple(aTuple) const b = encodeTuple(bTuple) + const actual = compare(a, b) + const expected = result assert.deepStrictEqual( - compare(a, b), - result, + actual, + expected, `compareTuple(${[TupleToString(aTuple), TupleToString(bTuple)].join( ", " )}) === compare(${[JSON.stringify(a), JSON.stringify(b)].join(", ")})` @@ -120,9 +122,9 @@ describe("codec", () => { const sample = () => { const x = sortedValues.length - const i = _.random(x - 1) - const j = _.random(x - 1) - const k = _.random(x - 1) + const i = randomInt(x - 1) + const j = randomInt(x - 1) + const k = randomInt(x - 1) const tuple: Tuple = [sortedValues[i], sortedValues[j], sortedValues[k]] const rank = i * x * x + j * x + k return { tuple, rank } @@ -136,8 +138,85 @@ describe("codec", () => { } }) }) -}) -function not(x: number) { - return x === 0 ? x : -1 * x -} + describe("codec options", () => { + it("Throws error if a value cannot be encoded", () => { + assert.throws(() => encodeValue("a\x00b", { disallow: ["\x00"] })) + }) + + it("Encodes and decodes with custom delimiter and escape characters", () => { + const options = { delimiter: ":", escape: "\\", disallow: [] } + const testCases = [ + "simple", + "with:delimiter", + "with\\escape", + "with\\:both", + "multiple::delimiters", + "multiple\\\\escapes", + "mixed\\::\\\\:cases", + ] + + for (const value of testCases) { + const encoded = encodeValue(value, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + value, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(value), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + } + }) + + it("Handles all encoding options configurations", () => { + const options = { delimiter: "\x01", escape: "\x02", disallow: ["\x00"] } + + const testCases = [ + "normal string", + "with spaces", + "with,punctuation!", + "with\nnewline", + "with\ttab", + ] + + for (const value of testCases) { + const encoded = encodeValue(value, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + value, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(value), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + } + }) + + it("Maintains proper escaping with nested delimiters", () => { + const options = { delimiter: ":", escape: "\\", disallow: [] } + const complexValue = "a:b\\:c\\\\:d" + const encoded = encodeValue(complexValue, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + complexValue, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(complexValue), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + }) + }) +}) diff --git a/src/helpers/codec.ts b/src/helpers/codec.ts index 431d6c2..e915de1 100644 --- a/src/helpers/codec.ts +++ b/src/helpers/codec.ts @@ -6,6 +6,29 @@ import { Tuple, Value } from "../storage/types" import { compare } from "./compare" import { UnreachableError } from "./Unreachable" +const defaultOptions: Required = { + delimiter: "\x00", + escape: "\x01", + disallow: [], +} + +export type CodecOptions = { + /** + * The delimiter between values in a tuple. + * @default "\x00" + */ + delimiter?: string + /** + * The escape byte. + * @default "\x01" + */ + escape?: string + /** + * Disallowed characters in strings. Throws an error if any are found while encoding. + */ + disallow?: string[] +} + // null < object < array < number < string < boolean export const encodingByte = { null: "b", @@ -20,10 +43,10 @@ export type EncodingType = keyof typeof encodingByte export const encodingRank = sortBy( Object.entries(encodingByte), - ([key, value]) => value + ([_key, value]) => value ).map(([key]) => key as EncodingType) -export function encodeValue(value: Value): string { +export function encodeValue(value: Value, options?: CodecOptions): string { if (value === null) { return encodingByte.null } @@ -31,16 +54,21 @@ export function encodeValue(value: Value): string { return encodingByte.boolean + value } if (typeof value === "string") { + for (const disallowed of options?.disallow ?? defaultOptions.disallow) { + if (value.includes(disallowed)) { + throw new Error(`Disallowed character found: ${disallowed}.`) + } + } return encodingByte.string + value } if (typeof value === "number") { return encodingByte.number + elen.encode(value) } if (Array.isArray(value)) { - return encodingByte.array + encodeTuple(value) + return encodingByte.array + encodeTuple(value, options) } if (typeof value === "object") { - return encodingByte.object + encodeObjectValue(value) + return encodingByte.object + encodeObjectValue(value, options) } throw new UnreachableError(value, "Unknown value type") } @@ -71,7 +99,7 @@ const decodeType = invert(encodingByte) as { [key: string]: keyof typeof encodingByte } -export function decodeValue(str: string): Value { +export function decodeValue(str: string, options?: CodecOptions): Value { const encoding: EncodingType = decodeType[str[0]] const rest = str.slice(1) @@ -88,62 +116,75 @@ export function decodeValue(str: string): Value { return elen.decode(rest) } if (encoding === "array") { - return decodeTuple(rest) + return decodeTuple(rest, options) } if (encoding === "object") { - return decodeObjectValue(rest) + return decodeObjectValue(rest, options) } throw new UnreachableError(encoding, "Invalid encoding byte") } -export function encodeTuple(tuple: Tuple) { +export function encodeTuple(tuple: Tuple, options?: CodecOptions) { + const delimiter = options?.delimiter ?? defaultOptions.delimiter + const escape = options?.escape ?? defaultOptions.escape + const reEscapeByte = new RegExp(escape, "g") + const reDelimiterByte = new RegExp(delimiter, "g") return tuple .map((value, i) => { - const encoded = encodeValue(value) + const encoded = encodeValue(value, options) return ( encoded // B -> BB or \ -> \\ - .replace(/\x01/g, "\x01\x01") + .replace(reEscapeByte, escape + escape) // A -> BA or x -> \x - .replace(/\x00/g, "\x01\x00") + "\x00" + .replace(reDelimiterByte, escape + delimiter) + delimiter ) }) .join("") } -export function decodeTuple(str: string) { +export function decodeTuple(str: string, options?: CodecOptions) { if (str === "") { return [] } + + const delimiter = options?.delimiter ?? defaultOptions.delimiter + const escape = options?.escape ?? defaultOptions.escape + // Capture all of the escaped BB and BA pairs and wait // til we find an exposed A. - const re = /(\x01(\x01|\x00)|\x00)/g + const matcher = new RegExp( + `(${escape}(${escape}|${delimiter})|${delimiter})`, + "g" + ) + const reEncodedEscape = new RegExp(escape + escape, "g") + const reEncodedDelimiter = new RegExp(escape + delimiter, "g") const tuple: Tuple = [] let start = 0 while (true) { - const match = re.exec(str) + const match = matcher.exec(str) if (match === null) { return tuple } - if (match[0][0] === "\x01") { - // If we match a \x01\x01 or \x01\x00 then keep going. + if (match[0][0] === escape) { + // If we match a escape+escape or escape+delimiter then keep going. continue } const end = match.index const escaped = str.slice(start, end) const unescaped = escaped // BB -> B - .replace(/\x01\x01/g, "\x01") + .replace(reEncodedEscape, escape) // BA -> A - .replace(/\x01\x00/g, "\x00") - const decoded = decodeValue(unescaped) + .replace(reEncodedDelimiter, delimiter) + const decoded = decodeValue(unescaped, options) tuple.push(decoded) // Skip over the \x00. start = end + 1 } } -function encodeObjectValue(obj: object) { +function encodeObjectValue(obj: object, options?: CodecOptions) { if (!isPlainObject(obj)) { throw new Error("Cannot serialize this object.") } @@ -152,11 +193,11 @@ function encodeObjectValue(obj: object) { // We allow undefined values in objects, but we want to strip them out before // serializing. .filter(([key, value]) => value !== undefined) - return encodeTuple(entries as Tuple) + return encodeTuple(entries as Tuple, options) } -function decodeObjectValue(str: string) { - const entries = decodeTuple(str) as Array<[string, Value]> +function decodeObjectValue(str: string, options?: CodecOptions) { + const entries = decodeTuple(str, options) as Array<[string, Value]> const obj = {} for (const [key, value] of entries) { obj[key] = value diff --git a/src/helpers/randomInt.ts b/src/helpers/randomInt.ts new file mode 100644 index 0000000..cc6851a --- /dev/null +++ b/src/helpers/randomInt.ts @@ -0,0 +1,3 @@ +export function randomInt(ceil: number): number { + return Math.floor(Math.random() * ceil) +}