diff --git a/app/client/src/pages/Editor/JSEditor/utils.ts b/app/client/src/pages/Editor/JSEditor/utils.ts index 965af60e021..d0ffd1d68c3 100644 --- a/app/client/src/pages/Editor/JSEditor/utils.ts +++ b/app/client/src/pages/Editor/JSEditor/utils.ts @@ -9,8 +9,14 @@ import { } from "./constants"; import { DropdownOption } from "design-system"; import { find, memoize } from "lodash"; -import { ECMA_VERSION, NodeTypes, SourceType } from "constants/ast"; -import { isLiteralNode, isPropertyNode, PropertyNode } from "workers/ast"; +import { + isLiteralNode, + isPropertyNode, + PropertyNode, + ECMA_VERSION, + NodeTypes, + SourceType, +} from "@shared/ast"; export interface JSActionDropdownOption extends DropdownOption { data: JSAction | null; diff --git a/app/client/src/workers/DependencyMap/utils.ts b/app/client/src/workers/DependencyMap/utils.ts index e7de093e27a..d566e918cce 100644 --- a/app/client/src/workers/DependencyMap/utils.ts +++ b/app/client/src/workers/DependencyMap/utils.ts @@ -1,7 +1,7 @@ import { flatten } from "lodash"; import toPath from "lodash/toPath"; import { EvalErrorTypes } from "utils/DynamicBindingUtils"; -import { extractIdentifiersFromCode } from "workers/ast"; +import { extractIdentifiersFromCode } from "@shared/ast"; import DataTreeEvaluator from "workers/DataTreeEvaluator"; import { convertPathToString } from "../evaluationUtils"; @@ -10,7 +10,10 @@ export const extractReferencesFromBinding = ( allPaths: Record, ): string[] => { const references: Set = new Set(); - const identifiers = extractIdentifiersFromCode(script); + const identifiers = extractIdentifiersFromCode( + script, + self?.evaluationVersion, + ); identifiers.forEach((identifier: string) => { // If the identifier exists directly, add it and return diff --git a/app/client/src/workers/JSObject/index.ts b/app/client/src/workers/JSObject/index.ts index 626972608cd..104166f178d 100644 --- a/app/client/src/workers/JSObject/index.ts +++ b/app/client/src/workers/JSObject/index.ts @@ -2,7 +2,7 @@ import { DataTree, DataTreeJSAction } from "entities/DataTree/dataTreeFactory"; import { isEmpty, set } from "lodash"; import { EvalErrorTypes } from "utils/DynamicBindingUtils"; import { JSUpdate, ParsedJSSubAction } from "utils/JSPaneUtils"; -import { isTypeOfFunction, parseJSObjectWithAST } from "workers/ast"; +import { isTypeOfFunction, parseJSObjectWithAST } from "@shared/ast"; import DataTreeEvaluator from "workers/DataTreeEvaluator"; import evaluateSync, { isFunctionAsync } from "workers/evaluate"; import { diff --git a/app/client/src/workers/Lint/utils.ts b/app/client/src/workers/Lint/utils.ts index a7104409c1a..8386c792e77 100644 --- a/app/client/src/workers/Lint/utils.ts +++ b/app/client/src/workers/Lint/utils.ts @@ -25,7 +25,7 @@ import { getLintErrorMessage, getLintSeverity, } from "components/editorComponents/CodeEditor/lintHelpers"; -import { ECMA_VERSION } from "constants/ast"; +import { ECMA_VERSION } from "@shared/ast"; import { IGNORED_LINT_ERRORS, SUPPORTED_WEB_APIS, diff --git a/app/shared/ast/index.ts b/app/shared/ast/index.ts new file mode 100644 index 00000000000..fdcd98ef345 --- /dev/null +++ b/app/shared/ast/index.ts @@ -0,0 +1,40 @@ +import { + ObjectExpression, + PropertyNode, + isIdentifierNode, + isVariableDeclarator, + isObjectExpression, + isLiteralNode, + isPropertyNode, + isPropertyAFunctionNode, + getAST, + extractIdentifiersFromCode, + getFunctionalParamsFromNode, + isTypeOfFunction, +} from "./src/index"; + +// constants +import { ECMA_VERSION, SourceType, NodeTypes } from "./src/constants"; + +// JSObjects +import { parseJSObjectWithAST } from "./src/jsObject"; + +// types or intefaces should be exported with type keyword, while enums can be exported like normal functions +export type { ObjectExpression, PropertyNode }; + +export { + isIdentifierNode, + isVariableDeclarator, + isObjectExpression, + isLiteralNode, + isPropertyNode, + isPropertyAFunctionNode, + getAST, + extractIdentifiersFromCode, + getFunctionalParamsFromNode, + isTypeOfFunction, + parseJSObjectWithAST, + ECMA_VERSION, + SourceType, + NodeTypes, +}; diff --git a/app/shared/ast/package.json b/app/shared/ast/package.json index b8e5ecdb883..6dda6cd7d07 100644 --- a/app/shared/ast/package.json +++ b/app/shared/ast/package.json @@ -12,19 +12,26 @@ "directory": "build" }, "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "test:unit": "$(npm bin)/jest -b --colors --no-cache --silent --coverage --collectCoverage=true --coverageDirectory='../../' --coverageReporters='json-summary'", + "test:jest": "$(npm bin)/jest --watch", "build": "rollup -c", "start": "rollup -c", "link-package": "yarn install && rollup -c && cd build && yarn link" }, "dependencies": { + "acorn": "^8.8.0", "acorn-walk": "^8.2.0", + "astring": "^1.7.5", + "lodash": "^4.17.21", "rollup": "^2.77.0", - "typescript": "4.5.5" + "typescript": "4.5.5", + "unescape-js": "^1.1.4" }, "devDependencies": { "@babel/preset-typescript": "^7.17.12", "@rollup/plugin-commonjs": "^22.0.0", + "@types/lodash": "^4.14.120", + "@types/jest": "^27.4.1", "@typescript-eslint/eslint-plugin": "^5.25.0", "@typescript-eslint/parser": "^5.25.0", "rollup-plugin-generate-package-json": "^3.2.0", diff --git a/app/shared/ast/rollup.config.js b/app/shared/ast/rollup.config.js index 03371d9130e..99cd12363ab 100644 --- a/app/shared/ast/rollup.config.js +++ b/app/shared/ast/rollup.config.js @@ -7,7 +7,7 @@ const packageJson = require("./package.json"); export default { // TODO: Figure out regex where each directory can be a separate module without having to manually add them - input: ["src/index.ts"], + input: ["./index.ts"], output: [ { file: packageJson.module, diff --git a/app/shared/ast/src/constants.ts b/app/shared/ast/src/constants.ts new file mode 100644 index 00000000000..7a2665485cb --- /dev/null +++ b/app/shared/ast/src/constants.ts @@ -0,0 +1,30 @@ +export const ECMA_VERSION = 11; + +/* Indicates the mode the code should be parsed in. +This influences global strict mode and parsing of import and export declarations. +*/ +export enum SourceType { + script = "script", + module = "module", +} + +// Each node has an attached type property which further defines +// what all properties can the node have. +// We will just define the ones we are working with +export enum NodeTypes { + Identifier = "Identifier", + AssignmentPattern = "AssignmentPattern", + Literal = "Literal", + Property = "Property", + // Declaration - https://github.com/estree/estree/blob/master/es5.md#declarations + FunctionDeclaration = "FunctionDeclaration", + ExportDefaultDeclaration = "ExportDefaultDeclaration", + VariableDeclarator = "VariableDeclarator", + // Expression - https://github.com/estree/estree/blob/master/es5.md#expressions + MemberExpression = "MemberExpression", + FunctionExpression = "FunctionExpression", + ArrowFunctionExpression = "ArrowFunctionExpression", + ObjectExpression = "ObjectExpression", + ArrayExpression = "ArrayExpression", + ThisExpression = "ThisExpression", +} diff --git a/app/shared/ast/src/index.test.ts b/app/shared/ast/src/index.test.ts new file mode 100644 index 00000000000..4b5459e2458 --- /dev/null +++ b/app/shared/ast/src/index.test.ts @@ -0,0 +1,413 @@ +import { extractIdentifiersFromCode } from "./index"; +import { parseJSObjectWithAST } from "./jsObject"; + +// describe("getAllIdentifiers", () => { +// it("works properly", () => { +// const cases: { script: string; expectedResults: string[] }[] = [ +// { +// // Entity reference +// script: "DirectTableReference", +// expectedResults: ["DirectTableReference"], +// }, +// { +// // One level nesting +// script: "TableDataReference.data", +// expectedResults: ["TableDataReference.data"], +// }, +// { +// // Deep nesting +// script: "TableDataDetailsReference.data.details", +// expectedResults: ["TableDataDetailsReference.data.details"], +// }, +// { +// // Deep nesting +// script: "TableDataDetailsMoreReference.data.details.more", +// expectedResults: ["TableDataDetailsMoreReference.data.details.more"], +// }, +// { +// // Deep optional chaining +// script: "TableDataOptionalReference.data?.details.more", +// expectedResults: ["TableDataOptionalReference.data"], +// }, +// { +// // Deep optional chaining with logical operator +// script: +// "TableDataOptionalWithLogical.data?.details.more || FallbackTableData.data", +// expectedResults: [ +// "TableDataOptionalWithLogical.data", +// "FallbackTableData.data", +// ], +// }, +// { +// // null coalescing +// script: "TableDataOptionalWithLogical.data ?? FallbackTableData.data", +// expectedResults: [ +// "TableDataOptionalWithLogical.data", +// "FallbackTableData.data", +// ], +// }, +// { +// // Basic map function +// script: "Table5.data.map(c => ({ name: c.name }))", +// expectedResults: ["Table5.data.map", "c.name"], +// }, +// { +// // Literal property search +// script: "Table6['data']", +// expectedResults: ["Table6"], +// }, +// { +// // Deep literal property search +// script: "TableDataOptionalReference['data'].details", +// expectedResults: ["TableDataOptionalReference"], +// }, +// { +// // Array index search +// script: "array[8]", +// expectedResults: ["array[8]"], +// }, +// { +// // Deep array index search +// script: "Table7.data[4]", +// expectedResults: ["Table7.data[4]"], +// }, +// { +// // Deep array index search +// script: "Table7.data[4].value", +// expectedResults: ["Table7.data[4].value"], +// }, +// { +// // string literal and array index search +// script: "Table['data'][9]", +// expectedResults: ["Table"], +// }, +// { +// // array index and string literal search +// script: "Array[9]['data']", +// expectedResults: ["Array[9]"], +// }, +// { +// // Index identifier search +// script: "Table8.data[row][name]", +// expectedResults: ["Table8.data", "row", "name"], +// }, +// { +// // Index identifier search with global +// script: "Table9.data[appsmith.store.row]", +// expectedResults: ["Table9.data", "appsmith.store.row"], +// }, +// { +// // Index literal with further nested lookups +// script: "Table10.data[row].name", +// expectedResults: ["Table10.data", "row"], +// }, +// { +// // IIFE and if conditions +// script: +// "(function(){ if(Table11.isVisible) { return Api1.data } else { return Api2.data } })()", +// expectedResults: ["Table11.isVisible", "Api1.data", "Api2.data"], +// }, +// { +// // Functions and arguments +// script: "JSObject1.run(Api1.data, Api2.data)", +// expectedResults: ["JSObject1.run", "Api1.data", "Api2.data"], +// }, +// { +// // IIFE - without braces +// script: `function() { +// const index = Input1.text + +// const obj = { +// "a": 123 +// } + +// return obj[index] + +// }()`, +// expectedResults: ["Input1.text"], +// }, +// { +// // IIFE +// script: `(function() { +// const index = Input2.text + +// const obj = { +// "a": 123 +// } + +// return obj[index] + +// })()`, +// expectedResults: ["Input2.text"], +// }, +// { +// // arrow IIFE - without braces - will fail +// script: `() => { +// const index = Input3.text + +// const obj = { +// "a": 123 +// } + +// return obj[index] + +// }()`, +// expectedResults: [], +// }, +// { +// // arrow IIFE +// script: `(() => { +// const index = Input4.text + +// const obj = { +// "a": 123 +// } + +// return obj[index] + +// })()`, +// expectedResults: ["Input4.text"], +// }, +// { +// // Direct object access +// script: `{ "a": 123 }[Input5.text]`, +// expectedResults: ["Input5.text"], +// }, +// { +// // Function declaration and default arguments +// script: `function run(apiData = Api1.data) { +// return apiData; +// }`, +// expectedResults: ["Api1.data"], +// }, +// { +// // Function declaration with arguments +// script: `function run(data) { +// return data; +// }`, +// expectedResults: [], +// }, +// { +// // anonymous function with variables +// script: `() => { +// let row = 0; +// const data = {}; +// while(row < 10) { +// data["test__" + row] = Table12.data[row]; +// row = row += 1; +// } +// }`, +// expectedResults: ["Table12.data"], +// }, +// { +// // function with variables +// script: `function myFunction() { +// let row = 0; +// const data = {}; +// while(row < 10) { +// data["test__" + row] = Table13.data[row]; +// row = row += 1; +// } +// }`, +// expectedResults: ["Table13.data"], +// }, +// { +// // expression with arithmetic operations +// script: `Table14.data + 15`, +// expectedResults: ["Table14.data"], +// }, +// { +// // expression with logical operations +// script: `Table15.data || [{}]`, +// expectedResults: ["Table15.data"], +// }, +// ]; + +// cases.forEach((perCase) => { +// const references = extractIdentifiersFromCode(perCase.script); +// expect(references).toStrictEqual(perCase.expectedResults); +// }); +// }); +// }); + +// describe("parseJSObjectWithAST", () => { +// it("parse js object", () => { +// const body = `{ +// myVar1: [], +// myVar2: {}, +// myFun1: () => { +// //write code here +// }, +// myFun2: async () => { +// //use async-await or promises +// } +// }`; +// const parsedObject = [ +// { +// key: "myVar1", +// value: "[]", +// type: "ArrayExpression", +// }, +// { +// key: "myVar2", +// value: "{}", +// type: "ObjectExpression", +// }, +// { +// key: "myFun1", +// value: "() => {}", +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// { +// key: "myFun2", +// value: "async () => {}", +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// ]; +// const resultParsedObject = parseJSObjectWithAST(body); +// expect(resultParsedObject).toStrictEqual(parsedObject); +// }); + +// it("parse js object with literal", () => { +// const body = `{ +// myVar1: [], +// myVar2: { +// "a": "app", +// }, +// myFun1: () => { +// //write code here +// }, +// myFun2: async () => { +// //use async-await or promises +// } +// }`; +// const parsedObject = [ +// { +// key: "myVar1", +// value: "[]", +// type: "ArrayExpression", +// }, +// { +// key: "myVar2", +// value: '{\n "a": "app"\n}', +// type: "ObjectExpression", +// }, +// { +// key: "myFun1", +// value: "() => {}", +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// { +// key: "myFun2", +// value: "async () => {}", +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// ]; +// const resultParsedObject = parseJSObjectWithAST(body); +// expect(resultParsedObject).toStrictEqual(parsedObject); +// }); + +// it("parse js object with variable declaration inside function", () => { +// const body = `{ +// myFun1: () => { +// const a = { +// conditions: [], +// requires: 1, +// testFunc: () => {}, +// testFunc2: function(){} +// }; +// }, +// myFun2: async () => { +// //use async-await or promises +// } +// }`; +// const parsedObject = [ +// { +// key: "myFun1", +// value: `() => { +// const a = { +// conditions: [], +// requires: 1, +// testFunc: () => {}, +// testFunc2: function () {} +// }; +// }`, +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// { +// key: "myFun2", +// value: "async () => {}", +// type: "ArrowFunctionExpression", +// arguments: [], +// }, +// ]; +// const resultParsedObject = parseJSObjectWithAST(body); +// expect(resultParsedObject).toStrictEqual(parsedObject); +// }); + +// it("parse js object with params of all types", () => { +// const body = `{ +// myFun2: async (a,b = Array(1,2,3),c = "", d = [], e = this.myVar1, f = {}, g = function(){}, h = Object.assign({}), i = String(), j = storeValue()) => { +// //use async-await or promises +// }, +// }`; + +// const parsedObject = [ +// { +// key: "myFun2", +// value: +// 'async (a, b = Array(1, 2, 3), c = "", d = [], e = this.myVar1, f = {}, g = function () {}, h = Object.assign({}), i = String(), j = storeValue()) => {}', +// type: "ArrowFunctionExpression", +// arguments: [ +// { +// paramName: "a", +// defaultValue: undefined, +// }, +// { +// paramName: "b", +// defaultValue: undefined, +// }, +// { +// paramName: "c", +// defaultValue: undefined, +// }, +// { +// paramName: "d", +// defaultValue: undefined, +// }, +// { +// paramName: "e", +// defaultValue: undefined, +// }, +// { +// paramName: "f", +// defaultValue: undefined, +// }, +// { +// paramName: "g", +// defaultValue: undefined, +// }, +// { +// paramName: "h", +// defaultValue: undefined, +// }, +// { +// paramName: "i", +// defaultValue: undefined, +// }, +// { +// paramName: "j", +// defaultValue: undefined, +// }, +// ], +// }, +// ]; +// const resultParsedObject = parseJSObjectWithAST(body); +// expect(resultParsedObject).toEqual(parsedObject); +// }); +// }); diff --git a/app/shared/ast/src/index.ts b/app/shared/ast/src/index.ts index 0036c97da26..171f08ecd7d 100644 --- a/app/shared/ast/src/index.ts +++ b/app/shared/ast/src/index.ts @@ -1,9 +1,352 @@ -const isNullOrWhitespace = (value: string) => - value === undefined || value === null || !value.trim(); -const isNullX = (value: string) => value === undefined || value === null; +import { parse, Node } from "acorn"; +import { ancestor } from "acorn-walk"; +import { ECMA_VERSION, NodeTypes } from "./constants"; +import { isFinite, isString } from "lodash"; +import { sanitizeScript } from "./utils"; -export type Animal = { +/* + * Valuable links: + * + * * ESTree spec: Javascript AST is called ESTree. + * Each es version has its md file in the repo to find features + * implemented and their node type + * https://github.com/estree/estree + * + * * Acorn: The parser we use to get the AST + * https://github.com/acornjs/acorn + * + * * Acorn walk: The walker we use to traverse the AST + * https://github.com/acornjs/acorn/tree/master/acorn-walk + * + * * AST Explorer: Helpful web tool to see ASTs and its parts + * https://astexplorer.net/ + * + */ + +type Pattern = IdentifierNode | AssignmentPatternNode; +type Expression = Node; +// doc: https://github.com/estree/estree/blob/master/es5.md#memberexpression +interface MemberExpressionNode extends Node { + type: NodeTypes.MemberExpression; + object: MemberExpressionNode | IdentifierNode; + property: IdentifierNode | LiteralNode; + computed: boolean; + // doc: https://github.com/estree/estree/blob/master/es2020.md#chainexpression + optional?: boolean; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#identifier +interface IdentifierNode extends Node { + type: NodeTypes.Identifier; name: string; - age: number; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#variabledeclarator +interface VariableDeclaratorNode extends Node { + type: NodeTypes.VariableDeclarator; + id: IdentifierNode; + init: Expression | null; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#functions +interface Function extends Node { + id: IdentifierNode | null; + params: Pattern[]; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#functiondeclaration +interface FunctionDeclarationNode extends Node, Function { + type: NodeTypes.FunctionDeclaration; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#functionexpression +interface FunctionExpressionNode extends Expression, Function { + type: NodeTypes.FunctionExpression; +} + +interface ArrowFunctionExpressionNode extends Expression, Function { + type: NodeTypes.ArrowFunctionExpression; +} + +export interface ObjectExpression extends Expression { + type: NodeTypes.ObjectExpression; + properties: Array; +} + +// doc: https://github.com/estree/estree/blob/master/es2015.md#assignmentpattern +interface AssignmentPatternNode extends Node { + type: NodeTypes.AssignmentPattern; + left: Pattern; +} + +// doc: https://github.com/estree/estree/blob/master/es5.md#literal +interface LiteralNode extends Node { + type: NodeTypes.Literal; + value: string | boolean | null | number | RegExp; +} + +// https://github.com/estree/estree/blob/master/es5.md#property +export interface PropertyNode extends Node { + type: NodeTypes.Property; + key: LiteralNode | IdentifierNode; + value: Node; + kind: "init" | "get" | "set"; +} + +/* We need these functions to typescript casts the nodes with the correct types */ +export const isIdentifierNode = (node: Node): node is IdentifierNode => { + return node.type === NodeTypes.Identifier; +}; + +const isMemberExpressionNode = (node: Node): node is MemberExpressionNode => { + return node.type === NodeTypes.MemberExpression; +}; + +export const isVariableDeclarator = ( + node: Node +): node is VariableDeclaratorNode => { + return node.type === NodeTypes.VariableDeclarator; +}; + +const isFunctionDeclaration = (node: Node): node is FunctionDeclarationNode => { + return node.type === NodeTypes.FunctionDeclaration; +}; + +const isFunctionExpression = (node: Node): node is FunctionExpressionNode => { + return node.type === NodeTypes.FunctionExpression; +}; + +export const isObjectExpression = (node: Node): node is ObjectExpression => { + return node.type === NodeTypes.ObjectExpression; +}; + +const isAssignmentPatternNode = (node: Node): node is AssignmentPatternNode => { + return node.type === NodeTypes.AssignmentPattern; +}; + +export const isLiteralNode = (node: Node): node is LiteralNode => { + return node.type === NodeTypes.Literal; +}; + +export const isPropertyNode = (node: Node): node is PropertyNode => { + return node.type === NodeTypes.Property; +}; + +export const isPropertyAFunctionNode = ( + node: Node +): node is ArrowFunctionExpressionNode | FunctionExpressionNode => { + return ( + node.type === NodeTypes.ArrowFunctionExpression || + node.type === NodeTypes.FunctionExpression + ); +}; + +const isArrayAccessorNode = (node: Node): node is MemberExpressionNode => { + return ( + isMemberExpressionNode(node) && + node.computed && + isLiteralNode(node.property) && + isFinite(node.property.value) + ); +}; + +const wrapCode = (code: string) => { + return ` + (function() { + return ${code} + }) + `; +}; + +export const getAST = (code: string) => + parse(code, { ecmaVersion: ECMA_VERSION }); + +/** + * An AST based extractor that fetches all possible identifiers in a given + * piece of code. We use this to get any references to the global entities in Appsmith + * and create dependencies on them. If the reference was updated, the given piece of code + * should run again. + * @param code: The piece of script where identifiers need to be extracted from + */ +export const extractIdentifiersFromCode = ( + code: string, + evaluationVersion: number +): string[] => { + // List of all identifiers found + const identifiers = new Set(); + // List of variables declared within the script. This will be removed from identifier list + const variableDeclarations = new Set(); + // List of functionalParams found. This will be removed from the identifier list + let functionalParams = new Set(); + let ast: Node = { end: 0, start: 0, type: "" }; + try { + const sanitizedScript = sanitizeScript(code, evaluationVersion); + /* wrapCode - Wrapping code in a function, since all code/script get wrapped with a function during evaluation. + Some syntax won't be valid unless they're at the RHS of a statement. + Since we're assigning all code/script to RHS during evaluation, we do the same here. + So that during ast parse, those errors are neglected. + */ + /* e.g. IIFE without braces + function() { return 123; }() -> is invalid + let result = function() { return 123; }() -> is valid + */ + const wrappedCode = wrapCode(sanitizedScript); + ast = getAST(wrappedCode); + } catch (e) { + if (e instanceof SyntaxError) { + // Syntax error. Ignore and return 0 identifiers + return []; + } + throw e; + } + + /* + * We do an ancestor walk on the AST to get all identifiers. Since we need to know + * what surrounds the identifier, ancestor walk will give that information in the callback + * doc: https://github.com/acornjs/acorn/tree/master/acorn-walk + */ + ancestor(ast, { + Identifier(node: Node, ancestors: Node[]) { + /* + * We are interested in identifiers. Due to the nature of AST, Identifier nodes can + * also be nested inside MemberExpressions. For deeply nested object references, there + * could be nesting of many MemberExpressions. To find the final reference, we will + * try to find the top level MemberExpression that does not have a MemberExpression parent. + * */ + let candidateTopLevelNode: IdentifierNode | MemberExpressionNode = + node as IdentifierNode; + let depth = ancestors.length - 2; // start "depth" with first parent + while (depth > 0) { + const parent = ancestors[depth]; + if ( + isMemberExpressionNode(parent) && + /* Member expressions that are "computed" (with [ ] search) + and the ones that have optional chaining ( a.b?.c ) + will be considered top level node. + We will stop looking for further parents */ + /* "computed" exception - isArrayAccessorNode + Member expressions that are array accessors with static index - [9] + will not be considered top level. + We will continue looking further. */ + (!parent.computed || isArrayAccessorNode(parent)) && + !parent.optional + ) { + candidateTopLevelNode = parent; + depth = depth - 1; + } else { + // Top level found + break; + } + } + if (isIdentifierNode(candidateTopLevelNode)) { + // If the node is an Identifier, just save that + identifiers.add(candidateTopLevelNode.name); + } else { + // For MemberExpression Nodes, we will construct a final reference string and then add + // it to the identifier list + const memberExpIdentifier = constructFinalMemberExpIdentifier( + candidateTopLevelNode + ); + identifiers.add(memberExpIdentifier); + } + }, + VariableDeclarator(node: Node) { + // keep a track of declared variables so they can be + // subtracted from the final list of identifiers + if (isVariableDeclarator(node)) { + variableDeclarations.add(node.id.name); + } + }, + FunctionDeclaration(node: Node) { + // params in function declarations are also counted as identifiers so we keep + // track of them and remove them from the final list of identifiers + if (!isFunctionDeclaration(node)) return; + functionalParams = new Set([ + ...functionalParams, + ...getFunctionalParamsFromNode(node), + ]); + }, + FunctionExpression(node: Node) { + // params in function experssions are also counted as identifiers so we keep + // track of them and remove them from the final list of identifiers + if (!isFunctionExpression(node)) return; + functionalParams = new Set([ + ...functionalParams, + ...getFunctionalParamsFromNode(node), + ]); + }, + }); + + // Remove declared variables and function params + variableDeclarations.forEach((variable) => identifiers.delete(variable)); + functionalParams.forEach((param) => identifiers.delete(param.paramName)); + + return Array.from(identifiers); +}; + +export type functionParams = { paramName: string; defaultValue: unknown }; + +export const getFunctionalParamsFromNode = ( + node: + | FunctionDeclarationNode + | FunctionExpressionNode + | ArrowFunctionExpressionNode, + needValue = false +): Set => { + const functionalParams = new Set(); + node.params.forEach((paramNode) => { + if (isIdentifierNode(paramNode)) { + functionalParams.add({ + paramName: paramNode.name, + defaultValue: undefined, + }); + } else if (isAssignmentPatternNode(paramNode)) { + if (isIdentifierNode(paramNode.left)) { + const paramName = paramNode.left.name; + if (!needValue) { + functionalParams.add({ paramName, defaultValue: undefined }); + } else { + // figure out how to get value of paramNode.right for each node type + // currently we don't use params value, hence skipping it + // functionalParams.add({ + // defaultValue: paramNode.right.value, + // }); + } + } + } + }); + return functionalParams; +}; + +const constructFinalMemberExpIdentifier = ( + node: MemberExpressionNode, + child = "" +): string => { + const propertyAccessor = getPropertyAccessor(node.property); + if (isIdentifierNode(node.object)) { + return `${node.object.name}${propertyAccessor}${child}`; + } else { + const propertyAccessor = getPropertyAccessor(node.property); + const nestedChild = `${propertyAccessor}${child}`; + return constructFinalMemberExpIdentifier(node.object, nestedChild); + } +}; + +const getPropertyAccessor = (propertyNode: IdentifierNode | LiteralNode) => { + if (isIdentifierNode(propertyNode)) { + return `.${propertyNode.name}`; + } else if (isLiteralNode(propertyNode) && isString(propertyNode.value)) { + // is string literal search a['b'] + return `.${propertyNode.value}`; + } else if (isLiteralNode(propertyNode) && isFinite(propertyNode.value)) { + // is array index search - a[9] + return `[${propertyNode.value}]`; + } +}; + +export const isTypeOfFunction = (type: string) => { + return ( + type === NodeTypes.ArrowFunctionExpression || + type === NodeTypes.FunctionExpression + ); }; -export { isNullOrWhitespace, isNullX }; diff --git a/app/shared/ast/src/jsObject/index.ts b/app/shared/ast/src/jsObject/index.ts new file mode 100644 index 00000000000..f35418ce98d --- /dev/null +++ b/app/shared/ast/src/jsObject/index.ts @@ -0,0 +1,75 @@ +import { Node } from "acorn"; +import { getAST } from "../index"; +import { generate } from "astring"; +import { simple } from "acorn-walk"; +import { + getFunctionalParamsFromNode, + isPropertyAFunctionNode, + isVariableDeclarator, + isObjectExpression, + PropertyNode, + functionParams, +} from "../index"; + +type JsObjectProperty = { + key: string; + value: string; + type: string; + arguments?: Array; +}; + +export const parseJSObjectWithAST = ( + jsObjectBody: string +): Array => { + /* + jsObjectVariableName value is added such actual js code would never name same variable name. + if the variable name will be same then also we won't have problem here as jsObjectVariableName will be last node in VariableDeclarator hence overriding the previous JSObjectProperties. + Keeping this just for sanity check if any caveat was missed. + */ + const jsObjectVariableName = + "____INTERNAL_JS_OBJECT_NAME_USED_FOR_PARSING_____"; + const jsCode = `var ${jsObjectVariableName} = ${jsObjectBody}`; + + const ast = getAST(jsCode); + + const parsedObjectProperties = new Set(); + let JSObjectProperties: Array = []; + + simple(ast, { + VariableDeclarator(node: Node) { + if ( + isVariableDeclarator(node) && + node.id.name === jsObjectVariableName && + node.init && + isObjectExpression(node.init) + ) { + JSObjectProperties = node.init.properties; + } + }, + }); + + JSObjectProperties.forEach((node) => { + let params = new Set(); + const propertyNode = node; + let property: JsObjectProperty = { + key: generate(propertyNode.key), + value: generate(propertyNode.value), + type: propertyNode.value.type, + }; + + if (isPropertyAFunctionNode(propertyNode.value)) { + // if in future we need default values of each param, we could implement that in getFunctionalParamsFromNode + // currently we don't consume it anywhere hence avoiding to calculate that. + params = getFunctionalParamsFromNode(propertyNode.value); + property = { + ...property, + arguments: [...params], + }; + } + + // here we use `generate` function to convert our AST Node to JSCode + parsedObjectProperties.add(property); + }); + + return [...parsedObjectProperties]; +}; diff --git a/app/shared/ast/src/typings/unescape-js/index.d.ts b/app/shared/ast/src/typings/unescape-js/index.d.ts new file mode 100644 index 00000000000..cfd13f93108 --- /dev/null +++ b/app/shared/ast/src/typings/unescape-js/index.d.ts @@ -0,0 +1 @@ +declare module "unescape-js"; diff --git a/app/shared/ast/src/utils.ts b/app/shared/ast/src/utils.ts new file mode 100644 index 00000000000..3a184df4dea --- /dev/null +++ b/app/shared/ast/src/utils.ts @@ -0,0 +1,11 @@ +import unescapeJS from "unescape-js"; + +const beginsWithLineBreakRegex = /^\s+|\s+$/; + +export function sanitizeScript(js: string, evaluationVersion: number) { + // We remove any line breaks from the beginning of the script because that + // makes the final function invalid. We also unescape any escaped characters + // so that eval can happen + const trimmedJS = js.replace(beginsWithLineBreakRegex, ""); + return evaluationVersion > 1 ? trimmedJS : unescapeJS(trimmedJS); +} diff --git a/app/shared/ast/tsconfig.json b/app/shared/ast/tsconfig.json index 5438deea911..5c0f356a63a 100644 --- a/app/shared/ast/tsconfig.json +++ b/app/shared/ast/tsconfig.json @@ -32,6 +32,6 @@ "baseUrl": "./src", "noFallthroughCasesInSwitch": true }, - "include": ["./src/**/*", "index.d.ts"], + "include": ["./src/**/*", "index.d.ts", "index.ts"], "exclude": ["node_modules", "build"] } diff --git a/app/shared/ast/yarn.lock b/app/shared/ast/yarn.lock index 2303058c2a5..0a3c9f0ed0f 100644 --- a/app/shared/ast/yarn.lock +++ b/app/shared/ast/yarn.lock @@ -300,11 +300,24 @@ dependencies: "@types/istanbul-lib-report" "*" +"@types/jest@^27.4.1": + version "27.5.2" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.2.tgz#ec49d29d926500ffb9fd22b84262e862049c026c" + integrity sha512-mpT8LJJ4CMeeahobofYWIjFo0xonRS/HfxnVEPMPFSQdGUt1uHCnoPT7Zhb+sjDU2wz0oKV0OLUR0WzrHNgfeA== + dependencies: + jest-matcher-utils "^27.0.0" + pretty-format "^27.0.0" + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== +"@types/lodash@^4.14.120": + version "4.14.182" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.182.tgz#05301a4d5e62963227eaafe0ce04dd77c54ea5c2" + integrity sha512-/THyiqyQAP9AfARo4pF+aCGcyiQ94tX/Is2I7HofNRqoYLgN1PBoOWu2/zTA5zMxzP5EFutMtWtGAFRKUe961Q== + "@types/node@*": version "18.0.6" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.0.6.tgz#0ba49ac517ad69abe7a1508bc9b3a5483df9d5d7" @@ -412,6 +425,16 @@ acorn-walk@^8.2.0: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== +acorn@^8.8.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" + integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -426,11 +449,21 @@ ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" +ansi-styles@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" + integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== + array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== +astring@^1.7.5: + version "1.8.3" + resolved "https://registry.yarnpkg.com/astring/-/astring-1.8.3.tgz#1a0ae738c7cc558f8e5ddc8e3120636f5cebcb85" + integrity sha512-sRpyiNrx2dEYIMmUXprS8nlpRg2Drs8m9ElX9vVEXaCB4XEAJhKfs7IcX0IwShjuOAjLR6wzIrgoptz1n19i1A== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" @@ -531,6 +564,11 @@ detect-indent@^5.0.0: resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" integrity sha512-rlpvsxUtM0PQvy9iZe640/IWwWYyBsTApREbA1pHOpmOUIl9MkP/U4z7vTtg4Oaojvqhxt7sdufnT0EzGaR31g== +diff-sequences@^27.5.1: + version "27.5.1" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" + integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ== + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -805,6 +843,31 @@ is-reference@^1.2.1: dependencies: "@types/estree" "*" +jest-diff@^27.5.1: + version "27.5.1" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def" + integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw== + dependencies: + chalk "^4.0.0" + diff-sequences "^27.5.1" + jest-get-type "^27.5.1" + pretty-format "^27.5.1" + +jest-get-type@^27.5.1: + version "27.5.1" + resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1" + integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw== + +jest-matcher-utils@^27.0.0: + version "27.5.1" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab" + integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw== + dependencies: + chalk "^4.0.0" + jest-diff "^27.5.1" + jest-get-type "^27.5.1" + pretty-format "^27.5.1" + jest-util@^27.0.0: version "27.5.1" resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-27.5.1.tgz#3ba9771e8e31a0b85da48fe0b0891fb86c01c2f9" @@ -858,7 +921,7 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" -lodash@4.x: +lodash@4.x, lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -1010,11 +1073,25 @@ pkg-dir@^4.1.0: dependencies: find-up "^4.0.0" +pretty-format@^27.0.0, pretty-format@^27.5.1: + version "27.5.1" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" + integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== + dependencies: + ansi-regex "^5.0.1" + ansi-styles "^5.0.0" + react-is "^17.0.1" + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +react-is@^17.0.1: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" + integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== + read-pkg@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc" @@ -1147,6 +1224,11 @@ spdx-license-ids@^3.0.0: resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.11.tgz#50c0d8c40a14ec1bf449bae69a0ea4685a9d9f95" integrity sha512-Ctl2BrFiM0X3MANYgj3CkygxhRmr9mi6xhejbdO960nF6EDJApTYpn0BQnDKlnNBULKiCN1n3w9EBkHK8ZWg+g== +string.fromcodepoint@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/string.fromcodepoint/-/string.fromcodepoint-0.2.1.tgz#8d978333c0bc92538f50f383e4888f3e5619d653" + integrity sha512-n69H31OnxSGSZyZbgBlvYIXlrMhJQ0dQAX1js1QDhpaUH6zmU3QYlj07bCwCNlPOu3oRXIubGPl2gDGnHsiCqg== + supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" @@ -1226,6 +1308,13 @@ typescript@4.5.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== +unescape-js@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/unescape-js/-/unescape-js-1.1.4.tgz#4bc6389c499cb055a98364a0b3094e1c3d5da395" + integrity sha512-42SD8NOQEhdYntEiUQdYq/1V/YHwr1HLwlHuTJB5InVVdOSbgI6xu8jK5q65yIzuFCfczzyDF/7hbGzVbyCw0g== + dependencies: + string.fromcodepoint "^0.2.1" + universalify@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"