support Apache Arrow as a normalized data representation

package.json

@@ -56,6 +56,7 @@
     "@types/node": "^20.5.0",
     "@typescript-eslint/eslint-plugin": "^7.2.0",
     "@typescript-eslint/parser": "^7.2.0",
+    "apache-arrow": "^16.0.2",
     "c8": "^9.1.0",
     "canvas": "^2.0.0",
     "d3-geo-projection": "^4.0.0",

src/mark.d.ts

@@ -32,6 +32,7 @@ export type TipPointer = "x" | "y" | "xy";
  *
  * - an array, typed array, or other iterable
  * - an object with a length property and indexed values
+ * - an Apache Arrow Table
  */
 export type Data = Iterable<any> | ArrayLike<any>;
 

src/mark.js

@@ -2,7 +2,7 @@ import {channelDomain, createChannels, valueObject} from "./channel.js";
 import {defined} from "./defined.js";
 import {maybeFacetAnchor} from "./facet.js";
 import {maybeClip, maybeNamed, maybeValue} from "./options.js";
-import {arrayify, isDomainSort, isObject, isOptions, keyword, range, singleton} from "./options.js";
+import {arrayify, isArrowTable, isDomainSort, isObject, isOptions, keyword, range, singleton} from "./options.js";
 import {project} from "./projection.js";
 import {styles} from "./style.js";
 import {basic, initializer} from "./transforms/basic.js";
@@ -87,10 +87,10 @@ export class Mark {
     }
   }
   initialize(facets, facetChannels, plotOptions) {
-    let data = arrayify(this.data);
+    let data = dataify(this.data);
     if (facets === undefined && data != null) facets = [range(data)];
     const originalFacets = facets;
-    if (this.transform != null) ({facets, data} = this.transform(data, facets, plotOptions)), (data = arrayify(data));
+    if (this.transform != null) ({facets, data} = this.transform(data, facets, plotOptions)), (data = dataify(data));
     if (facets !== undefined) facets.original = originalFacets; // needed to read facetChannels
     const channels = createChannels(this.channels, data);
     if (this.sort != null) channelDomain(data, facets, channels, facetChannels, this.sort); // mutates facetChannels!
@@ -130,6 +130,10 @@ export class Mark {
   }
 }
 
+function dataify(data) {
+  return isArrowTable(data) ? data : arrayify(data);
+}
+
 export function marks(...marks) {
   marks.plot = Mark.prototype.plot;
   return marks;

src/options.js

@@ -7,14 +7,36 @@ import {timeInterval, utcInterval} from "./time.js";
 export const TypedArray = Object.getPrototypeOf(Uint8Array);
 const objectToString = Object.prototype.toString;
 
+function isArray(value) {
+  return value instanceof Array || value instanceof TypedArray;
+}
+
+function isNumberArray(value) {
+  return value instanceof TypedArray && !isBigIntArray(value);
+}
+
+function isNumberType(type) {
+  return type?.prototype instanceof TypedArray && !isBigIntType(type);
+}
+
+function isBigIntArray(value) {
+  return value instanceof BigInt64Array || value instanceof BigUint64Array;
+}
+
+function isBigIntType(type) {
+  return type === BigInt64Array || type === BigUint64Array;
+}
+
 // If a reindex is attached to the data, channel values expressed as arrays will
 // be reindexed when the channels are instantiated. See exclusiveFacets.
 export const reindex = Symbol("reindex");
 
 export function valueof(data, value, type) {
   const valueType = typeof value;
   return valueType === "string"
-    ? maybeTypedMap(data, field(value), type)
+    ? isArrowTable(data)
+      ? maybeTypedArrowify(data.getChild(value), type)
+      : maybeTypedMap(data, field(value), type)
     : valueType === "function"
     ? maybeTypedMap(data, value, type)
     : valueType === "number" || value instanceof Date || valueType === "boolean"
@@ -29,19 +51,25 @@ function maybeTake(values, index) {
 }
 
 function maybeTypedMap(data, f, type) {
-  return map(data, type?.prototype instanceof TypedArray ? floater(f) : f, type);
+  return map(data, isNumberType(type) ? floater(f) : f, type); // coerce maybe BigInt to Number to avoid error
 }
 
 function maybeTypedArrayify(data, type) {
   return type === undefined
     ? arrayify(data) // preserve undefined type
     : data instanceof type
     ? data
-    : type.prototype instanceof TypedArray && !(data instanceof TypedArray)
+    : isNumberType(type) && !isNumberArray(data)
     ? type.from(data, coerceNumber)
     : type.from(data);
 }
 
+function maybeTypedArrowify(vector, type = Array) {
+  return type === Array && isArrowDateType(vector.type)
+    ? coerceDates(vector.toArray())
+    : maybeTypedArrayify(vector.toArray(), type);
+}
+
 function floater(f) {
   return (d, i) => coerceNumber(f(d, i));
 }
@@ -70,7 +98,7 @@ export function percentile(reduce) {
 
 // If the values are specified as a typed array, no coercion is required.
 export function coerceNumbers(values) {
-  return values instanceof TypedArray ? values : map(values, coerceNumber, Float64Array);
+  return isNumberArray(values) ? values : map(values, coerceNumber, Float64Array);
 }
 
 // Unlike Mark’s number, here we want to convert null and undefined to NaN since
@@ -95,7 +123,7 @@ export function coerceDate(x) {
     ? x
     : typeof x === "string"
     ? isoParse(x)
-    : x == null || isNaN((x = +x))
+    : x == null || isNaN((x = Number(x))) // allow conversion from BigInt
     ? undefined
     : new Date(x);
 }
@@ -132,7 +160,7 @@ export function keyword(input, name, allowed) {
 
 // Promotes the specified data to an array as needed.
 export function arrayify(values) {
-  if (values == null || values instanceof Array || values instanceof TypedArray) return values;
+  if (values == null || isArray(values)) return values;
   switch (values.type) {
     case "FeatureCollection":
       return values.features;
@@ -235,20 +263,15 @@ export function maybeZ({z, fill, stroke} = {}) {
 
 // Returns a Uint32Array with elements [0, 1, 2, … data.length - 1].
 export function range(data) {
-  const n = data.length;
+  const n = isArray(data) ? data.length : data.numRows;
   const r = new Uint32Array(n);
   for (let i = 0; i < n; ++i) r[i] = i;
   return r;
 }
 
-// Returns a filtered range of data given the test function.
-export function where(data, test) {
-  return range(data).filter((i) => test(data[i], i, data));
-}
-
 // Returns an array [values[index[0]], values[index[1]], …].
 export function take(values, index) {
-  return map(index, (i) => values[i], values.constructor);
+  return isArray(values) ? map(index, (i) => values[i], values.constructor) : map(index, (i) => values.at(i));
 }
 
 // If f does not take exactly one argument, wraps it in a function that uses take.
@@ -575,3 +598,25 @@ export function maybeClip(clip) {
   else if (clip != null) clip = keyword(clip, "clip", ["frame", "sphere"]);
   return clip;
 }
+
+// https://github.com/observablehq/stdlib/blob/746ca2e69135df6178e4f3a17244def35d8d6b20/src/arrow.js#L4C1-L17C1
+export function isArrowTable(value) {
+  return (
+    value &&
+    typeof value.getChild === "function" &&
+    typeof value.toArray === "function" &&
+    value.schema &&
+    Array.isArray(value.schema.fields)
+  );
+}
+
+// Apache Arrow now represents dates as numbers. We currently only support
+// implicit coercion to JavaScript Date objects when the numbers represent
+// milliseconds since Unix epoch.
+function isArrowDateType(type) {
+  return (
+    (type.typeId === 8 || // date
+      type.typeId === 10) && // timestamp
+    type.unit === 1 // millisecond
+  );
+}