Merge pull request #2461 from IDEMSInternational/refactor/app-data-ev…

…aluator

refactor: app-data-evaluator

packages/shared/README.md

@@ -37,7 +37,7 @@ import { TemplatedData } from "packages/shared"
 In addition, to avoid compiler errors thrown by non-browser shared methods, explicit paths should be included to import only the supported files as required
 
 ```ts
-import { AppStringEvaluator } from "packages/shared/src/models/appStringEvaluator/appStringEvaluator";
+import { AppDataEvaluator } from "packages/shared/src/models/appDataEvaluator/appDataEvaluator";
 import { TemplatedData } from "packages/shared/src/models/templatedData/templatedData";
 ```
 

packages/shared/src/models/appDataEvaluator/appDataEvaluator.spec.ts

@@ -0,0 +1,42 @@
+import { AppDataEvaluator } from "./appDataEvaluator";
+
+describe("App String Evaluator - String Replacement", () => {
+  const evaluator = new AppDataEvaluator();
+
+  evaluator.setExecutionContext({
+    row: { first_name: "Ada", last_name: "Lovelace", number_1: 1, number_2: 2 },
+  });
+
+  it("Hello @row.first_name @row.last_name)", () => {
+    expect(evaluator.evaluate("Hello @row.first_name @row.last_name")).toEqual(
+      "Hello Ada Lovelace"
+    );
+  });
+  it("{nested:[@row.first_name]}", () => {
+    expect(evaluator.evaluate({ nested: ["@row.first_name"] })).toEqual({ nested: ["Ada"] });
+  });
+});
+
+describe("App String Evaluator - JS Evaluate", () => {
+  const evaluator = new AppDataEvaluator();
+
+  evaluator.setExecutionContext({
+    row: { first_name: "Ada", last_name: "Lovelace", number_1: 1, number_2: 2 },
+  });
+
+  it("@row.number_1 > @row.number_2", () => {
+    expect(evaluator.evaluate("@row.number_1 > @row.number_2")).toEqual(false);
+  });
+
+  it("@row.first_name === 'Ada'", () => {
+    expect(evaluator.evaluate("@row.first_name === 'Ada'")).toEqual(true);
+  });
+
+  it("@row.first_name.length", () => {
+    expect(evaluator.evaluate("@row.first_name.length")).toEqual(3);
+  });
+  it("@row.first_name==='Ada' ? 'It is Ada' : 'It is not Ada'", () => {
+    const res = evaluator.evaluate("@row.first_name==='Ada' ? 'It is Ada' : 'It is not Ada'");
+    expect(res).toEqual("It is Ada");
+  });
+});

packages/shared/src/models/appDataEvaluator/appDataEvaluator.ts

@@ -0,0 +1,109 @@
+import { JSEvaluator } from "../jsEvaluator/jsEvaluator";
+import { TemplatedData } from "../templatedData/templatedData";
+import { isObjectLiteral } from "../../utils/object-utils";
+import { addJSDelimeters } from "../../utils/delimiters";
+
+/** Variable context is stored in namespaces, e.g. `{item:{key:'value'},field:{key:'value'}}` */
+type IContext = { [nameSpace: string]: { [field: string]: any } };
+
+/**
+ * Utility class to allow evaluation of app data that contain a mix of context expressions,
+ * JavaScript and static strings. It combines both TemplatedData and jsEvaluator models
+ *
+ * @example
+ * ```
+ * const evaluator = new AppDataEvaluator()
+ *
+ * evaluator.setExecutionContext({
+ *  row:{
+ *    number_1:1,
+ *    number_2:2
+ *  }
+ * })
+ *
+ * const expression = `@row.number_1 > @row.number_2 ? '1 is greater' : '2 is greater'`
+ * evaluator.evaluate(expression1)
+ * // 2 is greater
+ * ```
+ * */
+export class AppDataEvaluator {
+  private templatedData = new TemplatedData();
+  private jsEvaluator = new JSEvaluator();
+  constructor(private context = {}) {
+    this.setExecutionContext(context);
+  }
+  public setExecutionContext(context: IContext) {
+    this.context = context;
+    this.templatedData.updateContext(context);
+  }
+  public updateExecutionContext(update: IContext) {
+    this.setExecutionContext({ ...this.context, update });
+  }
+
+  /**
+   * Evaluate app expression in two stages
+   *  1) Replace any instances of context variables with values
+   * e.g. `@row.number_1 > @row.number_2 ? '1 is greater' : '2 is greater'
+   * ->   `1 > 2 ? '1 is greater' : '2 is greater'
+   *
+   * 2) Attempt evaluation in a JS context
+   * -> '2 is greater'
+   *
+   * @param data Input data to evaluated. This can be any data type, including
+   * nested objects or arrays. All string entries within will be evaulated
+   */
+  public evaluate(data: any) {
+    if (typeof data === "string") {
+      return this.evaluateExpression(data);
+    }
+    if (Array.isArray(data)) {
+      const evaluated = [];
+      for (const el of data) {
+        evaluated.push(this.evaluate(el));
+      }
+      return evaluated;
+    }
+    if (isObjectLiteral(data)) {
+      // create a new object to avoid changing initial data
+      const evaluated = {};
+      for (const [key, value] of Object.entries(data)) {
+        evaluated[key] = this.evaluate(value);
+      }
+      return evaluated;
+    }
+    return data;
+  }
+
+  /**
+   * Evaluate app string expression in two stages
+   * 1) Attempt to parse as a JavaScript expression
+   * Any variables referenced with `@` syntax will be converted to `this.`
+   * and evaluated with available context variables
+   * @example
+   * ```js
+   * evaluateExpression(`@row.number_1 > @row.number_2 ? 'bigger' : 'smaller'`)
+   * // intermediate interpretation
+   * `this.row.number_1 > this.row.number_2 ? 'bigger' : 'smaller'`
+   * // output
+   * "smaller"
+   * ```
+   *
+   * 2) Perform string replacement of individual variables
+   * @example
+   * ```js
+   * evaluateExpression(`Hello @row.first_name`)
+   * ```
+   * // output
+   * "Hello Ada"
+   */
+  private evaluateExpression(expression: string) {
+    try {
+      const jsDelimited = addJSDelimeters(expression, Object.keys(this.context));
+      const evaluated = this.jsEvaluator.evaluate(jsDelimited, this.context);
+      return evaluated;
+    } catch (error) {
+      const parsed = this.templatedData.parse(expression);
+      return parsed;
+    }
+  }
+}

packages/shared/src/models/dataPipe/operators/appendColumns.ts

@@ -1,5 +1,5 @@
 import { DataFrame, toJSON } from "danfojs";
-import { AppStringEvaluator } from "../../appStringEvaluator/appStringEvaluator";
+import { AppDataEvaluator } from "../../appDataEvaluator/appDataEvaluator";
 import BaseOperator from "./base";
 import { parseStringValue } from "../../../utils";
 
@@ -24,7 +24,7 @@ class AppendColumnsOperator extends BaseOperator {
     return arg.key && arg.valueExpression !== undefined;
   }
   apply() {
-    const evaluator = new AppStringEvaluator();
+    const evaluator = new AppDataEvaluator();
     for (const { key, valueExpression } of this.args_list) {
       const rows = toJSON(this.df) as any[];
       const appendValues = rows.map((row) => {

packages/shared/src/models/index.ts

@@ -1,4 +1,4 @@
-export * from "./appStringEvaluator/appStringEvaluator";
+export * from "./appDataEvaluator/appDataEvaluator";
 export * from "./templatedData/templatedData";
 export * from "./jsEvaluator/jsEvaluator";
 

packages/shared/src/utils/delimiters.spec.ts

@@ -7,7 +7,7 @@ interface IDelimitedTestData {
 }
 
 const prefixes = ["row"];
-const delimitedTests: IDelimitedTestData[] = [
+const stringDelimiterTests: IDelimitedTestData[] = [
   // basic
   {
     input: "Name: @row.first_name",
@@ -39,6 +39,80 @@ interface IParseTestData {
   delimited: string;
   extracted: ITemplatedStringVariable;
 }
+
+describe("addStringDelimiters", () => {
+  // Test individual string parsing
+  for (const testData of stringDelimiterTests) {
+    execTest(testData);
+  }
+
+  // Use a function wrapper to allow looping tests
+  function execTest(testData: IDelimitedTestData) {
+    const { input, delimited } = testData;
+
+    it(JSON.stringify(input), () => {
+      const parsedValue = Delimiters.addStringDelimiters(input, prefixes);
+      expect(parsedValue).toEqual(delimited);
+      process.nextTick(() => console.log(`      ${JSON.stringify(parsedValue)}\n`));
+      // NOTE - in case of errors additional tests can be carried out just on intermediate
+    });
+  }
+});
+
+const jsDelimterTests: IDelimitedTestData[] = [
+  // nested
+  {
+    input: "@[email protected]_name",
+    delimited: "this.row[this.row.first_name]",
+  },
+
+  // basic
+  {
+    input: "@row.first_name",
+    delimited: "this.row.first_name",
+  },
+  // expression
+  {
+    input: "@row.first_name === 'Ada'",
+    delimited: "this.row.first_name === 'Ada'",
+  },
+
+  // nested with braces
+  {
+    input: "@row.{@row.first_name}",
+    delimited: "this.row[this.row.first_name]",
+  },
+  // invalid (will need to be parsed as templated string)
+  {
+    input: "Hello @row.first_name @row.last_name",
+    delimited: "Hello this.row.first_name this.row.last_name",
+  },
+];
+
+interface IParseTestData {
+  delimited: string;
+  extracted: ITemplatedStringVariable;
+}
+
+describe("addJSDelimiters", () => {
+  // Test individual string parsing
+  for (const testData of jsDelimterTests) {
+    execTest(testData);
+  }
+
+  // Use a function wrapper to allow looping tests
+  function execTest(testData: IDelimitedTestData) {
+    const { input, delimited } = testData;
+
+    it(JSON.stringify(input), () => {
+      const parsedValue = Delimiters.addJSDelimeters(input, prefixes);
+      expect(parsedValue).toEqual(delimited);
+      process.nextTick(() => console.log(`      ${JSON.stringify(parsedValue)}\n`));
+      // NOTE - in case of errors additional tests can be carried out just on intermediate
+    });
+  }
+});
+
 const parseTests: IParseTestData[] = [
   // basic
   {
@@ -93,25 +167,6 @@ const parseTests: IParseTestData[] = [
   },
 ];
 
-describe("Converts non-delimited variables", () => {
-  // Test individual string parsing
-  for (const testData of delimitedTests) {
-    execTest(testData);
-  }
-
-  // Use a function wrapper to allow looping tests
-  function execTest(testData: IDelimitedTestData) {
-    const { input, delimited } = testData;
-
-    it(JSON.stringify(input), () => {
-      const parsedValue = Delimiters.addStringDelimiters(input, prefixes);
-      expect(parsedValue).toEqual(delimited);
-      process.nextTick(() => console.log(`      ${JSON.stringify(parsedValue)}\n`));
-      // NOTE - in case of errors additional tests can be carried out just on intermediate
-    });
-  }
-});
-
 describe("Parses delimiters", () => {
   // Test individual string parsing
   for (const testData of parseTests) {