Expanded BigInt support (#2)

Co-authored-by: Jason Paulos <[email protected]>

Author: robdmoore

.vscode/extensions.json

@@ -1,10 +1,6 @@
 {
-	// List of extensions which should be recommended for users of this workspace.
-	"recommendations": [
-		"dbaeumer.vscode-eslint",
-		"yzhang.markdown-all-in-one"
-	],
-	// List of extensions recommended by VS Code that should not be recommended for users of this workspace.
-	"unwantedRecommendations": [
-	]
+  // List of extensions which should be recommended for users of this workspace.
+  "recommendations": ["dbaeumer.vscode-eslint", "yzhang.markdown-all-in-one", "hbenl.vscode-mocha-test-adapter"],
+  // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
+  "unwantedRecommendations": []
 }

.vscode/settings.json

@@ -2,13 +2,13 @@
   "typescript.tsdk": "node_modules/typescript/lib",
   "files.eol": "\n",
   "editor.tabSize": 2,
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.formatOnSave": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": true,
+    "source.fixAll": "always"
   },
-  "cSpell.words": [
-    "instanceof",
-    "tsdoc",
-    "typeof",
-    "whatwg"
-  ]
+  "cSpell.words": ["instanceof", "tsdoc", "typeof", "whatwg"],
+  "mochaExplorer.files": "test/**/*.test.{ts,js}",
+  "mochaExplorer.require": ["ts-node/register", "tsconfig-paths/register"]
 }

README.md

@@ -74,7 +74,9 @@
     "karma-webpack": "latest",
     "lodash": "latest",
     "mocha": "latest",
-    "msgpack-test-js": "latest",
+    "msg-ext": "^1.0.1",
+    "msg-int64": "^0.1.1",
+    "msg-timestamp": "^1.0.1",
     "prettier": "latest",
     "rimraf": "latest",
     "ts-loader": "latest",

package-lock.json

@@ -1,6 +1,6 @@
 import { prettyByte } from "./utils/prettyByte";
 import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
-import { getInt64, getUint64, UINT32_MAX } from "./utils/int";
+import { IntMode, getInt64, getUint64, convertSafeIntegerToMode, UINT32_MAX } from "./utils/int";
 import { utf8Decode } from "./utils/utf8";
 import { createDataView, ensureUint8Array } from "./utils/typedArrays";
 import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder";
@@ -16,10 +16,17 @@ export type DecoderOptions<ContextType = undefined> = Readonly<
      * Depends on ES2020's {@link DataView#getBigInt64} and
      * {@link DataView#getBigUint64}.
      *
-     * Defaults to false.
+     * Defaults to false. If true, equivalent to intMode: IntMode.AS_ENCODED.
      */
     useBigInt64: boolean;
 
+    /**
+     * Allows for more fine-grained control of BigInt handling, overrides useBigInt64.
+     *
+     * Defaults to IntMode.AS_ENCODED if useBigInt64 is true or IntMode.UNSAFE_NUMBER otherwise.
+     */
+    intMode?: IntMode;
+
     /**
      * Maximum string length.
      *
@@ -194,7 +201,7 @@ const sharedCachedKeyDecoder = new CachedKeyDecoder();
 export class Decoder<ContextType = undefined> {
   private readonly extensionCodec: ExtensionCodecType<ContextType>;
   private readonly context: ContextType;
-  private readonly useBigInt64: boolean;
+  private readonly intMode: IntMode;
   private readonly maxStrLength: number;
   private readonly maxBinLength: number;
   private readonly maxArrayLength: number;
@@ -214,7 +221,7 @@ export class Decoder<ContextType = undefined> {
     this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType<ContextType>);
     this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined
 
-    this.useBigInt64 = options?.useBigInt64 ?? false;
+    this.intMode = options?.intMode ?? (options?.useBigInt64 ? IntMode.AS_ENCODED : IntMode.UNSAFE_NUMBER);
     this.maxStrLength = options?.maxStrLength ?? UINT32_MAX;
     this.maxBinLength = options?.maxBinLength ?? UINT32_MAX;
     this.maxArrayLength = options?.maxArrayLength ?? UINT32_MAX;
@@ -371,11 +378,11 @@ export class Decoder<ContextType = undefined> {
 
       if (headByte >= 0xe0) {
         // negative fixint (111x xxxx) 0xe0 - 0xff
-        object = headByte - 0x100;
+        object = this.convertNumber(headByte - 0x100);
       } else if (headByte < 0xc0) {
         if (headByte < 0x80) {
           // positive fixint (0xxx xxxx) 0x00 - 0x7f
-          object = headByte;
+          object = this.convertNumber(headByte);
         } else if (headByte < 0x90) {
           // fixmap (1000 xxxx) 0x80 - 0x8f
           const size = headByte - 0x80;
@@ -418,36 +425,28 @@ export class Decoder<ContextType = undefined> {
         object = this.readF64();
       } else if (headByte === 0xcc) {
         // uint 8
-        object = this.readU8();
+        object = this.convertNumber(this.readU8());
       } else if (headByte === 0xcd) {
         // uint 16
-        object = this.readU16();
+        object = this.convertNumber(this.readU16());
       } else if (headByte === 0xce) {
         // uint 32
-        object = this.readU32();
+        object = this.convertNumber(this.readU32());
       } else if (headByte === 0xcf) {
         // uint 64
-        if (this.useBigInt64) {
-          object = this.readU64AsBigInt();
-        } else {
-          object = this.readU64();
-        }
+        object = this.readU64();
       } else if (headByte === 0xd0) {
         // int 8
-        object = this.readI8();
+        object = this.convertNumber(this.readI8());
       } else if (headByte === 0xd1) {
         // int 16
-        object = this.readI16();
+        object = this.convertNumber(this.readI16());
       } else if (headByte === 0xd2) {
         // int 32
-        object = this.readI32();
+        object = this.convertNumber(this.readI32());
       } else if (headByte === 0xd3) {
         // int 64
-        if (this.useBigInt64) {
-          object = this.readI64AsBigInt();
-        } else {
-          object = this.readI64();
-        }
+        object = this.readI64();
       } else if (headByte === 0xd9) {
         // str 8
         const byteLength = this.lookU8();
@@ -692,6 +691,10 @@ export class Decoder<ContextType = undefined> {
     return this.extensionCodec.decode(data, extType, this.context);
   }
 
+  private convertNumber(value: number): number | bigint {
+    return convertSafeIntegerToMode(value, this.intMode);
+  }
+
   private lookU8() {
     return this.view.getUint8(this.pos);
   }
@@ -740,26 +743,14 @@ export class Decoder<ContextType = undefined> {
     return value;
   }
 
-  private readU64(): number {
-    const value = getUint64(this.view, this.pos);
-    this.pos += 8;
-    return value;
-  }
-
-  private readI64(): number {
-    const value = getInt64(this.view, this.pos);
-    this.pos += 8;
-    return value;
-  }
-
-  private readU64AsBigInt(): bigint {
-    const value = this.view.getBigUint64(this.pos);
+  private readU64(): number | bigint {
+    const value = getUint64(this.view, this.pos, this.intMode);
     this.pos += 8;
     return value;
   }
 
-  private readI64AsBigInt(): bigint {
-    const value = this.view.getBigInt64(this.pos);
+  private readI64(): number | bigint {
+    const value = getInt64(this.view, this.pos, this.intMode);
     this.pos += 8;
     return value;
   }

package.json

@@ -13,14 +13,14 @@ export type EncoderOptions<ContextType = undefined> = Partial<
     extensionCodec: ExtensionCodecType<ContextType>;
 
     /**
-     * Encodes bigint as Int64 or Uint64 if it's set to true.
-     * {@link forceIntegerToFloat} does not affect bigint.
+     * Encodes bigint as Int64 or Uint64 if it's set to true, regardless of the size of bigint number.
+     * {@link forceIntegerToFloat} does not affect bigint if this is enabled.
      * Depends on ES2020's {@link DataView#setBigInt64} and
      * {@link DataView#setBigUint64}.
      *
      * Defaults to false.
      */
-    useBigInt64: boolean;
+    forceBigIntToInt64: boolean;
 
     /**
      * The maximum depth in nested objects and arrays.
@@ -43,6 +43,7 @@ export type EncoderOptions<ContextType = undefined> = Partial<
      * Defaults to `false`. If enabled, it spends more time in encoding objects.
      */
     sortKeys: boolean;
+
     /**
      * If `true`, non-integer numbers are encoded in float32, not in float64 (the default).
      *
@@ -74,7 +75,7 @@ export type EncoderOptions<ContextType = undefined> = Partial<
 export class Encoder<ContextType = undefined> {
   private readonly extensionCodec: ExtensionCodecType<ContextType>;
   private readonly context: ContextType;
-  private readonly useBigInt64: boolean;
+  private readonly forceBigIntToInt64: boolean;
   private readonly maxDepth: number;
   private readonly initialBufferSize: number;
   private readonly sortKeys: boolean;
@@ -90,7 +91,7 @@ export class Encoder<ContextType = undefined> {
     this.extensionCodec = options?.extensionCodec ?? (ExtensionCodec.defaultCodec as ExtensionCodecType<ContextType>);
     this.context = (options as { context: ContextType } | undefined)?.context as ContextType; // needs a type assertion because EncoderOptions has no context property when ContextType is undefined
 
-    this.useBigInt64 = options?.useBigInt64 ?? false;
+    this.forceBigIntToInt64 = options?.forceBigIntToInt64 ?? false;
     this.maxDepth = options?.maxDepth ?? DEFAULT_MAX_DEPTH;
     this.initialBufferSize = options?.initialBufferSize ?? DEFAULT_INITIAL_BUFFER_SIZE;
     this.sortKeys = options?.sortKeys ?? false;
@@ -137,15 +138,9 @@ export class Encoder<ContextType = undefined> {
     } else if (typeof object === "boolean") {
       this.encodeBoolean(object);
     } else if (typeof object === "number") {
-      if (!this.forceIntegerToFloat) {
-        this.encodeNumber(object);
-      } else {
-        this.encodeNumberAsFloat(object);
-      }
+      this.encodeNumber(object);
     } else if (typeof object === "string") {
       this.encodeString(object);
-    } else if (this.useBigInt64 && typeof object === "bigint") {
-      this.encodeBigInt64(object);
     } else {
       this.encodeObject(object, depth);
     }
@@ -200,12 +195,10 @@ export class Encoder<ContextType = undefined> {
           // uint 32
           this.writeU8(0xce);
           this.writeU32(object);
-        } else if (!this.useBigInt64) {
+        } else {
           // uint 64
           this.writeU8(0xcf);
           this.writeU64(object);
-        } else {
-          this.encodeNumberAsFloat(object);
         }
       } else {
         if (object >= -0x20) {
@@ -223,12 +216,10 @@ export class Encoder<ContextType = undefined> {
           // int 32
           this.writeU8(0xd2);
           this.writeI32(object);
-        } else if (!this.useBigInt64) {
+        } else {
           // int 64
           this.writeU8(0xd3);
           this.writeI64(object);
-        } else {
-          this.encodeNumberAsFloat(object);
         }
       }
     } else {
@@ -248,7 +239,33 @@ export class Encoder<ContextType = undefined> {
     }
   }
 
-  private encodeBigInt64(object: bigint): void {
+  private encodeBigInt(object: bigint) {
+    if (this.forceBigIntToInt64) {
+      this.encodeBigIntAsInt64(object);
+    } else if (object >= 0) {
+      if (object < 0x100000000 || this.forceIntegerToFloat) {
+        // uint 32 or lower, or force to float
+        this.encodeNumber(Number(object));
+      } else if (object < BigInt("0x10000000000000000")) {
+        // uint 64
+        this.encodeBigIntAsInt64(object);
+      } else {
+        throw new Error(`Bigint is too large for uint64: ${object}`);
+      }
+    } else {
+      if (object >= -0x80000000 || this.forceIntegerToFloat) {
+        // int 32 or lower, or force to float
+        this.encodeNumber(Number(object));
+      } else if (object >= BigInt(-1) * BigInt("0x8000000000000000")) {
+        // int 64
+        this.encodeBigIntAsInt64(object);
+      } else {
+        throw new Error(`Bigint is too small for int64: ${object}`);
+      }
+    }
+  }
+
+  private encodeBigIntAsInt64(object: bigint): void {
     if (object >= BigInt(0)) {
       // uint 64
       this.writeU8(0xcf);
@@ -300,6 +317,10 @@ export class Encoder<ContextType = undefined> {
       this.encodeArray(object, depth);
     } else if (ArrayBuffer.isView(object)) {
       this.encodeBinary(object);
+    } else if (typeof object === "bigint") {
+      // this is here instead of in doEncode so that we can try encoding with an extension first,
+      // otherwise we would break existing extensions for bigints
+      this.encodeBigInt(object);
     } else if (typeof object === "object") {
       this.encodeMap(object as Record<string, unknown>, depth);
     } else {

src/Decoder.ts

@@ -9,6 +9,8 @@ import { decode, decodeMulti } from "./decode";
 export { decode, decodeMulti };
 import type { DecodeOptions } from "./decode";
 export type { DecodeOptions };
+import { IntMode } from './utils/int';
+export { IntMode };
 
 import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync";
 export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream };

src/Encoder.ts

@@ -1,6 +1,6 @@
 // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type
 import { DecodeError } from "./DecodeError";
-import { getInt64, setInt64 } from "./utils/int";
+import { IntMode, getInt64, setInt64 } from "./utils/int";
 
 export const EXT_TIMESTAMP = -1;
 
@@ -87,7 +87,7 @@ export function decodeTimestampToTimeSpec(data: Uint8Array): TimeSpec {
     case 12: {
       // timestamp 96 = { nsec32 (unsigned), sec64 (signed) }
 
-      const sec = getInt64(view, 4);
+      const sec = getInt64(view, 4, IntMode.UNSAFE_NUMBER);
       const nsec = view.getUint32(0);
       return { sec, nsec };
     }