[faat] deckgl-arrow-layers module (#2680)

* deckgl-arrow-layers-module

Signed-off-by: Ihor Dykhta <[email protected]>

bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js

@@ -142,8 +142,9 @@ class KeplerGlJupyter {
           },
           data: {
             fields: d.fields,
-            // rows: d.allData
-            ...(d.dataContainer instanceof ArrowDataContainer ? {cols: d.dataContainer._cols} : {rows: d.allData})
+            ...(d.dataContainer instanceof ArrowDataContainer
+              ? {cols: d.dataContainer._cols}
+              : {rows: d.allData})
           }
         })),
         config,
@@ -153,12 +154,7 @@ class KeplerGlJupyter {
   }
 }
 
-export function addDataConfigToKeplerGl({
-  data: inputData,
-  config,
-  options,
-  store
-}) {
+export function addDataConfigToKeplerGl({data: inputData, config, options, store}) {
   const data = inputData ? dataToDatasets(inputData) : [];
   log(data);
 

package.json

@@ -22,6 +22,7 @@
     "./src/styles",
     "./src/localization",
     "./src/deckgl-layers",
+    "./src/deckgl-arrow-layers",
     "./src/layers",
     "./src/schemas",
     "./src/table",

src/deckgl-arrow-layers/babel.config.js

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+const KeplerPackage = require('./package');
+
+const PRESETS = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-typescript'];
+const PLUGINS = [
+  ['@babel/plugin-transform-typescript', {isTSX: true, allowDeclareFields: true}],
+  '@babel/plugin-transform-modules-commonjs',
+  '@babel/plugin-transform-class-properties',
+  '@babel/plugin-transform-optional-chaining',
+  '@babel/plugin-transform-logical-assignment-operators',
+  '@babel/plugin-transform-nullish-coalescing-operator',
+  '@babel/plugin-transform-export-namespace-from',
+  [
+    '@babel/transform-runtime',
+    {
+      regenerator: true
+    }
+  ],
+  [
+    'search-and-replace',
+    {
+      rules: [
+        {
+          search: '__PACKAGE_VERSION__',
+          replace: KeplerPackage.version
+        }
+      ]
+    }
+  ]
+];
+const ENV = {
+  test: {
+    plugins: ['istanbul']
+  },
+  debug: {
+    sourceMaps: 'inline',
+    retainLines: true
+  }
+};
+
+module.exports = function babel(api) {
+  api.cache(true);
+
+  return {
+    presets: PRESETS,
+    plugins: PLUGINS,
+    env: ENV
+  };
+};

src/deckgl-arrow-layers/package.json

@@ -0,0 +1,61 @@
+{
+  "name": "@kepler.gl/deckgl-arrow-layers",
+  "author": "Shan He <[email protected]>",
+  "version": "3.0.0",
+  "description": "Deck.gl layers with GeoArrow and GeoParquet support",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "keywords": [
+    "babel",
+    "es6",
+    "react",
+    "webgl",
+    "visualization",
+    "deck.gl"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/keplergl/kepler.gl.git"
+  },
+  "scripts": {
+    "build": "rm -fr dist && babel src --out-dir dist --source-maps inline --extensions '.ts,.tsx,.js,.jsx' --ignore '**/*.d.ts'",
+    "build:umd": "NODE_OPTIONS=--openssl-legacy-provider webpack --config ./webpack/umd.js --progress --env.prod",
+    "build:types": "tsc --project ./tsconfig.production.json",
+    "prepublish": "babel-node ../../scripts/license-header/bin --license ../../FILE-HEADER && yarn build && yarn build:types",
+    "stab": "mkdir -p dist && touch dist/index.js"
+  },
+  "files": [
+    "dist",
+    "umd"
+  ],
+  "dependencies": {
+    "@geoarrow/geoarrow-js": "^0.3.0",
+    "@math.gl/core": "^4.0.0",
+    "@math.gl/polygon": "^4.0.0",
+    "@math.gl/types": "^4.0.0",
+    "apache-arrow": ">=15",
+    "threads": "^1.7.0"
+  },
+  "peerDependencies": {
+    "@deck.gl/aggregation-layers": "^8.9.27",
+    "@deck.gl/core": "^8.9.27",
+    "@deck.gl/geo-layers": "^8.9.27",
+    "@deck.gl/layers": "^8.9.27"
+  },
+  "nyc": {
+    "sourceMap": false,
+    "instrument": false
+  },
+  "maintainers": [
+    "Shan He <[email protected]>"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "volta": {
+    "node": "18.18.2",
+    "yarn": "4.4.0"
+  },
+  "packageManager": "[email protected]"
+}

src/deckgl-arrow-layers/src/constants.ts

@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+// deck.gl-community
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Enum holding GeoArrow extension type names
+ */
+export enum EXTENSION_NAME {
+  POINT = 'geoarrow.point',
+  LINESTRING = 'geoarrow.linestring',
+  POLYGON = 'geoarrow.polygon',
+  MULTIPOINT = 'geoarrow.multipoint',
+  MULTILINESTRING = 'geoarrow.multilinestring',
+  MULTIPOLYGON = 'geoarrow.multipolygon'
+}
+
+export const DEFAULT_COLOR: [number, number, number, number] = [0, 0, 0, 255];

src/deckgl-arrow-layers/src/index.ts

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+export {EXTENSION_NAME} from './constants';
+
+export {GeoArrowScatterplotLayer} from './layers/geo-arrow-scatterplot-layer';
+export {GeoArrowTextLayer} from './layers/geo-arrow-text-layer';
+export {GeoArrowArcLayer} from './layers/geo-arrow-arc-layer';

src/deckgl-arrow-layers/src/layers/geo-arrow-arc-layer.ts

@@ -0,0 +1,201 @@
+// deck.gl-community
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  CompositeLayer,
+  CompositeLayerProps,
+  DefaultProps,
+  GetPickingInfoParams,
+  Layer,
+  LayersList,
+  assert
+} from '@deck.gl/core/typed';
+import {ArcLayer} from '@deck.gl/layers/typed';
+import type {ArcLayerProps} from '@deck.gl/layers/typed';
+import * as arrow from 'apache-arrow';
+import * as ga from '@geoarrow/geoarrow-js';
+import {assignAccessor, extractAccessorsFromProps} from '../utils/utils';
+import {child} from '@geoarrow/geoarrow-js';
+import {GeoArrowExtraPickingProps, computeChunkOffsets, getPickingInfo} from '../utils/picking';
+import {ColorAccessor, FloatAccessor, GeoArrowPickingInfo} from '../types';
+import {validateAccessors} from '../utils/validate';
+
+/** All properties supported by GeoArrowArcLayer */
+export type GeoArrowArcLayerProps = Omit<
+  ArcLayerProps<any>,
+  | 'data'
+  | 'getSourcePosition'
+  | 'getTargetPosition'
+  | 'getSourceColor'
+  | 'getTargetColor'
+  | 'getWidth'
+  | 'getHeight'
+  | 'getTilt'
+> &
+  _GeoArrowArcLayerProps &
+  CompositeLayerProps;
+
+/** Properties added by GeoArrowArcLayer */
+type _GeoArrowArcLayerProps = {
+  data: arrow.Table;
+
+  /**
+   * Method called to retrieve the source position of each object.
+   */
+  getSourcePosition: ga.vector.PointVector;
+
+  /**
+   * Method called to retrieve the target position of each object.
+   */
+  getTargetPosition: ga.vector.PointVector;
+
+  /**
+   * The rgba color is in the format of `[r, g, b, [a]]`.
+   * @default [0, 0, 0, 255]
+   */
+  getSourceColor?: ColorAccessor;
+
+  /**
+   * The rgba color is in the format of `[r, g, b, [a]]`.
+   * @default [0, 0, 0, 255]
+   */
+  getTargetColor?: ColorAccessor;
+
+  /**
+   * The line width of each object, in units specified by `widthUnits`.
+   * @default 1
+   */
+  getWidth?: FloatAccessor;
+
+  /**
+   * Multiplier of layer height. `0` will make the layer flat.
+   * @default 1
+   */
+  getHeight?: FloatAccessor;
+
+  /**
+   * Use to tilt the arc to the side if you have multiple arcs with the same source and target positions.
+   * @default 0
+   */
+  getTilt?: FloatAccessor;
+
+  /**
+   * If `true`, validate the arrays provided (e.g. chunk lengths)
+   * @default true
+   */
+  _validate?: boolean;
+};
+
+// Remove data from the upstream default props
+const {
+  data: _data,
+  getSourcePosition: _getSourcePosition,
+  getTargetPosition: _getTargetPosition,
+  ..._defaultProps
+} = ArcLayer.defaultProps;
+
+// Default props added by us
+const ourDefaultProps = {
+  _validate: true
+};
+
+// @ts-expect-error
+const defaultProps: DefaultProps<GeoArrowArcLayerProps> = {
+  ..._defaultProps,
+  ...ourDefaultProps
+};
+
+export class GeoArrowArcLayer<ExtraProps extends object = object> extends CompositeLayer<
+  GeoArrowArcLayerProps & ExtraProps
+> {
+  static defaultProps = defaultProps;
+  static layerName = 'GeoArrowArcLayer';
+
+  getPickingInfo(
+    params: GetPickingInfoParams & {
+      sourceLayer: {props: GeoArrowExtraPickingProps};
+    }
+  ): GeoArrowPickingInfo {
+    return getPickingInfo(params, this.props.data);
+  }
+
+  renderLayers(): Layer<object> | LayersList | null {
+    return this._renderLayersPoint();
+  }
+
+  _renderLayersPoint(): Layer<object> | LayersList | null {
+    const {
+      data: table,
+      getSourcePosition: sourcePosition,
+      getTargetPosition: targetPosition
+    } = this.props;
+
+    if (this.props._validate) {
+      validateAccessors(this.props, table);
+
+      // Note: below we iterate over table batches anyways, so this layer won't
+      // work as-is if data/table is null
+      assert(ga.vector.isPointVector(sourcePosition));
+      assert(ga.vector.isPointVector(targetPosition));
+    }
+
+    // Exclude manually-set accessors
+    const [accessors, otherProps] = extractAccessorsFromProps(this.props, [
+      'getSourcePosition',
+      'getTargetPosition'
+    ]);
+    const tableOffsets = computeChunkOffsets(table.data);
+
+    const layers: ArcLayer<any>[] = [];
+    for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) {
+      const sourceData = sourcePosition.data[recordBatchIdx];
+      const sourceValues = child.getPointChild(sourceData).values;
+      const targetData = targetPosition.data[recordBatchIdx];
+      const targetValues = child.getPointChild(targetData).values;
+
+      const props: ArcLayerProps<any> = {
+        // Note: because this is a composite layer and not doing the rendering
+        // itself, we still have to pass in our defaultProps
+        ...ourDefaultProps,
+        ...otherProps,
+
+        // used for picking purposes
+        recordBatchIdx,
+        tableOffsets,
+
+        id: `${this.props.id}-geoarrow-arc-${recordBatchIdx}`,
+        data: {
+          // @ts-expect-error passed through to enable use by function accessors
+          data: table.batches[recordBatchIdx],
+          length: sourceData.length,
+          attributes: {
+            getSourcePosition: {
+              value: sourceValues,
+              size: sourceData.type.listSize
+            },
+            getTargetPosition: {
+              value: targetValues,
+              size: targetData.type.listSize
+            }
+          }
+        }
+      };
+
+      for (const [propName, propInput] of Object.entries(accessors)) {
+        assignAccessor({
+          props,
+          propName,
+          propInput,
+          chunkIdx: recordBatchIdx
+        });
+      }
+
+      const SubLayerClass = this.getSubLayerClass('geo-arrow-arc-layer', ArcLayer);
+      const layer = new SubLayerClass(this.getSubLayerProps(props));
+      layers.push(layer);
+    }
+
+    return layers;
+  }
+}