feat: Extraction of AST Logic (#15892)

* POC for Shared AST Logic using Yarn Symlinks * fix: preinstall script for bundling shared packages * Merge commit * fix: updated the script to link, unlink the package as shared dep * fix: updated dependencies * Add a post-install script and fix yarn.lock file * Remove commented code * fix: added verification script, readme, moved scripts to shared * Extraction of AST Logic into shared/ast folder * Add jest test script * Replace hardcoded ast Logic use with Shared AST module * Replace parse code with getAST Co-authored-by: Aman Agarwal <[email protected]>
appsmithorg · Aug 23, 2022 · 3de00c6 · 3de00c6
1 parent 5cf06cd
commit 3de00c6
Show file tree

Hide file tree

Showing 15 changed files with 1,035 additions and 17 deletions.
diff --git a/app/client/src/pages/Editor/JSEditor/utils.ts b/app/client/src/pages/Editor/JSEditor/utils.ts
@@ -9,8 +9,14 @@ import {
 } from "./constants";
 import { DropdownOption } from "design-system";
 import { find, memoize } from "lodash";
-import { ECMA_VERSION, NodeTypes, SourceType } from "constants/ast";
-import { isLiteralNode, isPropertyNode, PropertyNode } from "workers/ast";
+import {
+  isLiteralNode,
+  isPropertyNode,
+  PropertyNode,
+  ECMA_VERSION,
+  NodeTypes,
+  SourceType,
+} from "@shared/ast";
 
 export interface JSActionDropdownOption extends DropdownOption {
   data: JSAction | null;

diff --git a/app/client/src/workers/DependencyMap/utils.ts b/app/client/src/workers/DependencyMap/utils.ts
@@ -1,7 +1,7 @@
 import { flatten } from "lodash";
 import toPath from "lodash/toPath";
 import { EvalErrorTypes } from "utils/DynamicBindingUtils";
-import { extractIdentifiersFromCode } from "workers/ast";
+import { extractIdentifiersFromCode } from "@shared/ast";
 import DataTreeEvaluator from "workers/DataTreeEvaluator";
 import { convertPathToString } from "../evaluationUtils";
 
@@ -10,7 +10,10 @@ export const extractReferencesFromBinding = (
   allPaths: Record<string, true>,
 ): string[] => {
   const references: Set<string> = new Set<string>();
-  const identifiers = extractIdentifiersFromCode(script);
+  const identifiers = extractIdentifiersFromCode(
+    script,
+    self?.evaluationVersion,
+  );
 
   identifiers.forEach((identifier: string) => {
     // If the identifier exists directly, add it and return

diff --git a/app/client/src/workers/JSObject/index.ts b/app/client/src/workers/JSObject/index.ts
@@ -2,7 +2,7 @@ import { DataTree, DataTreeJSAction } from "entities/DataTree/dataTreeFactory";
 import { isEmpty, set } from "lodash";
 import { EvalErrorTypes } from "utils/DynamicBindingUtils";
 import { JSUpdate, ParsedJSSubAction } from "utils/JSPaneUtils";
-import { isTypeOfFunction, parseJSObjectWithAST } from "workers/ast";
+import { isTypeOfFunction, parseJSObjectWithAST } from "@shared/ast";
 import DataTreeEvaluator from "workers/DataTreeEvaluator";
 import evaluateSync, { isFunctionAsync } from "workers/evaluate";
 import {

diff --git a/app/client/src/workers/Lint/utils.ts b/app/client/src/workers/Lint/utils.ts
@@ -25,7 +25,7 @@ import {
   getLintErrorMessage,
   getLintSeverity,
 } from "components/editorComponents/CodeEditor/lintHelpers";
-import { ECMA_VERSION } from "constants/ast";
+import { ECMA_VERSION } from "@shared/ast";
 import {
   IGNORED_LINT_ERRORS,
   SUPPORTED_WEB_APIS,

diff --git a/app/shared/ast/index.ts b/app/shared/ast/index.ts
@@ -0,0 +1,40 @@
+import {
+  ObjectExpression,
+  PropertyNode,
+  isIdentifierNode,
+  isVariableDeclarator,
+  isObjectExpression,
+  isLiteralNode,
+  isPropertyNode,
+  isPropertyAFunctionNode,
+  getAST,
+  extractIdentifiersFromCode,
+  getFunctionalParamsFromNode,
+  isTypeOfFunction,
+} from "./src/index";
+
+// constants
+import { ECMA_VERSION, SourceType, NodeTypes } from "./src/constants";
+
+// JSObjects
+import { parseJSObjectWithAST } from "./src/jsObject";
+
+// types or intefaces should be exported with type keyword, while enums can be exported like normal functions
+export type { ObjectExpression, PropertyNode };
+
+export {
+  isIdentifierNode,
+  isVariableDeclarator,
+  isObjectExpression,
+  isLiteralNode,
+  isPropertyNode,
+  isPropertyAFunctionNode,
+  getAST,
+  extractIdentifiersFromCode,
+  getFunctionalParamsFromNode,
+  isTypeOfFunction,
+  parseJSObjectWithAST,
+  ECMA_VERSION,
+  SourceType,
+  NodeTypes,
+};
diff --git a/app/shared/ast/package.json b/app/shared/ast/package.json
@@ -12,19 +12,26 @@
     "directory": "build"
   },
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test:unit": "$(npm bin)/jest -b --colors --no-cache --silent --coverage --collectCoverage=true --coverageDirectory='../../' --coverageReporters='json-summary'",
+    "test:jest": "$(npm bin)/jest --watch",
     "build": "rollup -c",
     "start": "rollup -c",
     "link-package": "yarn install && rollup -c && cd build && yarn link"
   },
   "dependencies": {
+    "acorn": "^8.8.0",
     "acorn-walk": "^8.2.0",
+    "astring": "^1.7.5",
+    "lodash": "^4.17.21",
     "rollup": "^2.77.0",
-    "typescript": "4.5.5"
+    "typescript": "4.5.5",
+    "unescape-js": "^1.1.4"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.17.12",
     "@rollup/plugin-commonjs": "^22.0.0",
+    "@types/lodash": "^4.14.120",
+    "@types/jest": "^27.4.1",
     "@typescript-eslint/eslint-plugin": "^5.25.0",
     "@typescript-eslint/parser": "^5.25.0",
     "rollup-plugin-generate-package-json": "^3.2.0",

diff --git a/app/shared/ast/rollup.config.js b/app/shared/ast/rollup.config.js
@@ -7,7 +7,7 @@ const packageJson = require("./package.json");
 
 export default {
   // TODO: Figure out regex where each directory can be a separate module without having to manually add them
-  input: ["src/index.ts"],
+  input: ["./index.ts"],
   output: [
     {
       file: packageJson.module,

diff --git a/app/shared/ast/src/constants.ts b/app/shared/ast/src/constants.ts
@@ -0,0 +1,30 @@
+export const ECMA_VERSION = 11;
+
+/* Indicates the mode the code should be parsed in. 
+This influences global strict mode and parsing of import and export declarations.
+*/
+export enum SourceType {
+  script = "script",
+  module = "module",
+}
+
+// Each node has an attached type property which further defines
+// what all properties can the node have.
+// We will just define the ones we are working with
+export enum NodeTypes {
+  Identifier = "Identifier",
+  AssignmentPattern = "AssignmentPattern",
+  Literal = "Literal",
+  Property = "Property",
+  // Declaration - https://github.com/estree/estree/blob/master/es5.md#declarations
+  FunctionDeclaration = "FunctionDeclaration",
+  ExportDefaultDeclaration = "ExportDefaultDeclaration",
+  VariableDeclarator = "VariableDeclarator",
+  // Expression - https://github.com/estree/estree/blob/master/es5.md#expressions
+  MemberExpression = "MemberExpression",
+  FunctionExpression = "FunctionExpression",
+  ArrowFunctionExpression = "ArrowFunctionExpression",
+  ObjectExpression = "ObjectExpression",
+  ArrayExpression = "ArrayExpression",
+  ThisExpression = "ThisExpression",
+}