fix: Refactor entities based on AST parsing logic

app/rts/package.json

@@ -36,7 +36,6 @@
   "dependencies": {
     "express-validator": "^6.14.2",
     "http-status-codes": "^2.2.0",
-    "morgan": "^1.10.0",
     "supertest": "^6.2.4",
     "tsc-alias": "^1.7.0"
   }

app/rts/src/server.ts

@@ -1,7 +1,6 @@
 import http from "http";
 import path from "path";
 import express from "express";
-import morgan from "morgan";
 import { Server } from "socket.io";
 import log, { LogLevelDesc } from "loglevel";
 import { VERSION as buildVersion } from "./version"; // release version of the api
@@ -49,8 +48,6 @@ const io = new Server(server, {
 // Initializing Sockets
 initializeSockets(io);
 
-//Track perf metrics for each call
-app.use(morgan('tiny'));
 // parse incoming json requests
 app.use(express.json({ limit: "5mb" }));
 // Initializing Routes

app/rts/src/test/server.test.ts

@@ -50,6 +50,44 @@ const entityRefactor = [
     isJSObject: true,
     evalVersion: 2,
   },
+  {
+    script:
+      "//   ApiNever  \n function ApiNever(abc) {let foo = \"I'm getting data from ApiNever but don't rename this string\" +     ApiNever.data; \n if(true) { return ApiNever }}",
+    oldName: "ApiNever.data",
+    newName: "ApiNever.input",
+    isJSObject: false,
+    evalVersion: 2,
+  },
+  {
+    script:
+      "//   ApiNever  \n function ApiNever(abc) {let foo = \"I'm getting data from ApiNever but don't rename this string\" +     ApiNever.data; \n if(true) { return ApiNever }}",
+    oldName: "ApiNever.dat",
+    newName: "ApiNever.input",
+    isJSObject: false,
+    evalVersion: 2,
+  },
+  {
+    script: "\tApiNever.data",
+    oldName: "ApiNever",
+    newName: "ApiForever",
+    isJSObject: false,
+    evalVersion: 2,
+  },
+  {
+    script: "ApiNever.data + ApiNever.data",
+    oldName: "ApiNever",
+    newName: "ApiForever",
+    isJSObject: false,
+    evalVersion: 2,
+  },
+  {
+    script:
+      'export default {\n\tmyVar1: [],\n\tmyVar2: {},\n\tmyFun1: () => {\n\t\t//write code here\n\t\t// ApiNever.text\n\t\treturn "ApiNever.text" + ApiNever.text\n\t},\n\tmyFun2: async () => {\n\t\t//use async-await or promises\n\t\t// ApiNever.text\n\t\treturn "ApiNever.text" + ApiNever.text\n\t}\n}',
+    oldName: "ApiNever",
+    newName: "ApiForever",
+    isJSObject: true,
+    evalVersion: 2,
+  },
 ];
 
 afterAll((done) => {
@@ -124,6 +162,29 @@ describe("AST tests", () => {
             "export default {\n\tmyVar1: [],\n\tmyVar2: {},\n\t\tsearch: () => {\n\t\tif(Input1.text.length==0){\n\t\t\treturn select_repair_db.data\n\t\t}\n\t\telse{\n\t\t\treturn(select_repair_db.data.filter(word => word.cust_name.toLowerCase().includes(Input1.text.toLowerCase())))\n\t\t}\n\t},\n}",
           refactorCount: 2,
         },
+        {
+          script:
+            "//   ApiNever  \n function ApiNever(abc) {let foo = \"I'm getting data from ApiNever but don't rename this string\" +     ApiNever.input; \n if(true) { return ApiNever }}",
+          refactorCount: 1,
+        },
+        {
+          script:
+            "//   ApiNever  \n function ApiNever(abc) {let foo = \"I'm getting data from ApiNever but don't rename this string\" +     ApiNever.data; \n if(true) { return ApiNever }}",
+          refactorCount: 0,
+        },
+        {
+          script: "\tApiForever.data",
+          refactorCount: 1,
+        },
+        {
+          script: "ApiForever.data + ApiForever.data",
+          refactorCount: 2,
+        },
+        {
+          script:
+            'export default {\n\tmyVar1: [],\n\tmyVar2: {},\n\tmyFun1: () => {\n\t\t//write code here\n\t\t// ApiNever.text\n\t\treturn "ApiNever.text" + ApiForever.text\n\t},\n\tmyFun2: async () => {\n\t\t//use async-await or promises\n\t\t// ApiNever.text\n\t\treturn "ApiNever.text" + ApiForever.text\n\t}\n}',
+          refactorCount: 2,
+        },
       ];
 
       await supertest(app)

app/rts/yarn.lock

@@ -951,13 +951,6 @@ [email protected], base64id@~2.0.0:
   resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6"
   integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==
 
-basic-auth@~2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a"
-  integrity sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==
-  dependencies:
-    safe-buffer "5.1.2"
-
 binary-extensions@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
@@ -1283,7 +1276,7 @@ denque@^1.4.1:
   resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
   integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==
 
-[email protected], depd@~2.0.0:
+[email protected]:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df"
   integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==
@@ -2475,17 +2468,6 @@ mongodb@^3.6.4:
   optionalDependencies:
     saslprep "^1.0.0"
 
-morgan@^1.10.0:
-  version "1.10.0"
-  resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.10.0.tgz#091778abc1fc47cd3509824653dae1faab6b17d7"
-  integrity sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==
-  dependencies:
-    basic-auth "~2.0.1"
-    debug "2.6.9"
-    depd "~2.0.0"
-    on-finished "~2.3.0"
-    on-headers "~1.0.2"
-
 [email protected]:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -2560,18 +2542,6 @@ [email protected]:
   dependencies:
     ee-first "1.1.1"
 
-on-finished@~2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947"
-  integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=
-  dependencies:
-    ee-first "1.1.1"
-
-on-headers@~1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.2.tgz#772b0ae6aaa525c399e489adfad90c403eb3c28f"
-  integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==
-
 [email protected], once@^1.3.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
@@ -2857,16 +2827,16 @@ run-parallel@^1.1.9:
   dependencies:
     queue-microtask "^1.2.2"
 
-[email protected], safe-buffer@~5.1.0, safe-buffer@~5.1.1:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
-  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
-
 [email protected], safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
 "safer-buffer@>= 2.1.2 < 3":
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/AnalyticsEvents.java

@@ -60,6 +60,13 @@ public enum AnalyticsEvents {
     // Events to log execution time
     GIT_SERIALIZE_APP_RESOURCES_TO_LOCAL_FILE,
     GIT_DESERIALIZE_APP_RESOURCES_FROM_FILE,
+
+    // Entity refactor related events
+    REFACTOR_JSOBJECT,
+    REFACTOR_ACTION,
+    REFACTOR_JSACTION,
+    REFACTOR_WIDGET,
+
     INVITE_USERS_TO_USER_GROUPS,
     REMOVE_USERS_FROM_USER_GROUPS,
     ASSIGNED_TO_PERMISSION_GROUP,

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/helpers/MustacheHelper.java

@@ -3,6 +3,7 @@
 import com.appsmith.external.models.ActionConfiguration;
 import com.appsmith.external.models.EntityDependencyNode;
 import com.appsmith.external.models.EntityReferenceType;
+import com.appsmith.external.models.MustacheBindingToken;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.text.StringEscapeUtils;
 import org.springframework.beans.BeanWrapper;
@@ -23,7 +24,6 @@
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-import java.util.function.Function;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -79,12 +79,12 @@ public class MustacheHelper {
      * should give the original template back. The tokens are split such that alternative strings in the list are plain
      * text and the others are mustache interpolations.
      */
-    public static List<String> tokenize(String template) {
+    public static List<MustacheBindingToken> tokenize(String template) {
         if (!StringUtils.hasLength(template)) {
             return Collections.emptyList();
         }
 
-        List<String> tokens = new ArrayList<>();
+        List<MustacheBindingToken> tokens = new ArrayList<>();
 
         int length = template.length();
 
@@ -100,6 +100,7 @@ public static List<String> tokenize(String template) {
         int braceDepth = 0;
 
         StringBuilder currentToken = new StringBuilder().append(template.charAt(0));
+        int currentTokenStartIndex = 0;
 
         // The parser is implemented as a pointer (marked by `i`) that loops over each character in the template string.
         // There's majorly two states for the parser, plain-text-mode and mustache-mode, with the current state
@@ -118,8 +119,9 @@ public static List<String> tokenize(String template) {
                     isInsideMustache = true;
                     // Remove the `{` added to the builder.
                     currentToken.deleteCharAt(currentToken.length() - 1);
-                    clearAndPushToken(currentToken, tokens);
+                    clearAndPushToken(currentToken, currentTokenStartIndex, tokens);
                     currentToken.append(prevChar);
+                    currentTokenStartIndex = i - 1;
                     braceDepth = 2;
                 }
 
@@ -157,7 +159,7 @@ public static List<String> tokenize(String template) {
                     --braceDepth;
                     currentToken.append(currentChar);
                     if (prevChar == '}' && braceDepth <= 0) {
-                        clearAndPushToken(currentToken, tokens);
+                        clearAndPushToken(currentToken, currentTokenStartIndex, tokens);
                         isInsideMustache = false;
                     }
 
@@ -171,7 +173,7 @@ public static List<String> tokenize(String template) {
         }
 
         if (currentToken.length() > 0) {
-            tokens.add(currentToken.toString());
+            tokens.add(new MustacheBindingToken(currentToken.toString(), currentTokenStartIndex, true));
         }
 
         return tokens;
@@ -185,39 +187,39 @@ public static List<String> tokenize(String template) {
      * @return A Set of strings that serve as replacement keys, with the surrounding double braces stripped and then
      * trimmed.
      */
-    public static Set<String> extractMustacheKeys(String template) {
-        Set<String> keys = new HashSet<>();
+    public static Set<MustacheBindingToken> extractMustacheKeys(String template) {
+        Set<MustacheBindingToken> keys = new HashSet<>();
 
-        for (String token : tokenize(template)) {
-            if (token.startsWith("{{") && token.endsWith("}}")) {
+        for (MustacheBindingToken token : tokenize(template)) {
+            if (token.getValue().startsWith("{{") && token.getValue().endsWith("}}")) {
                 // Allowing empty tokens to be added, to be compatible with the previous `extractMustacheKeys` method.
                 // Calling `.trim()` before adding because Mustache compiler strips keys in the template before looking
                 // up a value. Addresses https://www.notion.so/appsmith/Bindings-with-a-space-at-the-start-fail-to-execute-properly-in-the-API-pane-2eb65d5c6064466b9ef059fa01ef3261
-                keys.add(token.substring(2, token.length() - 2).trim());
+                keys.add(new MustacheBindingToken(token.getValue().substring(2, token.getValue().length() - 2), (token.getStartIndex() + 2), false));
             }
         }
 
         return keys;
     }
 
     // For prepared statements we should extract the bindings in order in a list and include duplicate bindings as well.
-    public static List<String> extractMustacheKeysInOrder(String template) {
-        List<String> keys = new ArrayList<>();
+    public static List<MustacheBindingToken> extractMustacheKeysInOrder(String template) {
+        List<MustacheBindingToken> keys = new ArrayList<>();
 
-        for (String token : tokenize(template)) {
-            if (token.startsWith("{{") && token.endsWith("}}")) {
+        for (MustacheBindingToken token : tokenize(template)) {
+            if (token.getValue().startsWith("{{") && token.getValue().endsWith("}}")) {
                 // Allowing empty tokens to be added, to be compatible with the previous `extractMustacheKeys` method.
                 // Calling `.trim()` before adding because Mustache compiler strips keys in the template before looking
                 // up a value. Addresses https://www.notion.so/appsmith/Bindings-with-a-space-at-the-start-fail-to-execute-properly-in-the-API-pane-2eb65d5c6064466b9ef059fa01ef3261
-                keys.add(token.substring(2, token.length() - 2).trim());
+                keys.add(new MustacheBindingToken(token.getValue().substring(2, token.getValue().length() - 2).trim(), (token.getStartIndex() + 2), false));
             }
         }
 
         return keys;
     }
 
-    public static Set<String> extractMustacheKeysFromFields(Object object) {
-        final Set<String> keys = new HashSet<>();
+    public static Set<MustacheBindingToken> extractMustacheKeysFromFields(Object object) {
+        final Set<MustacheBindingToken> keys = new HashSet<>();
 
         // Linearized recursive search. Instead of calling this function recursively for nested values, we add them to
         // the end of the queue and process them in a linear fashion. This strategy doesn't suffer from a stack overflow
@@ -252,9 +254,9 @@ public static Set<String> extractMustacheKeysFromFields(Object object) {
         return keys;
     }
 
-    private static void clearAndPushToken(StringBuilder tokenBuilder, List<String> tokenList) {
+    private static void clearAndPushToken(StringBuilder tokenBuilder, int tokenStartIndex, List<MustacheBindingToken> tokenList) {
         if (tokenBuilder.length() > 0) {
-            tokenList.add(tokenBuilder.toString());
+            tokenList.add(new MustacheBindingToken(tokenBuilder.toString(), tokenStartIndex, true));
             tokenBuilder.setLength(0);
         }
     }
@@ -322,11 +324,11 @@ public static <T> T renderFieldValues(T object, Map<String, String> context) {
     public static String render(String template, Map<String, String> keyValueMap) {
         final StringBuilder rendered = new StringBuilder();
 
-        for (String token : tokenize(template)) {
-            if (token.startsWith("{{") && token.endsWith("}}")) {
-                rendered.append(keyValueMap.get(token.substring(2, token.length() - 2).trim()));
+        for (MustacheBindingToken token : tokenize(template)) {
+            if (token.getValue().startsWith("{{") && token.getValue().endsWith("}}")) {
+                rendered.append(keyValueMap.get(token.getValue().substring(2, token.getValue().length() - 2).trim()));
             } else {
-                rendered.append(token);
+                rendered.append(token.getValue());
             }
         }
 
@@ -500,26 +502,26 @@ public static Set<String> getPossibleParents(String mustacheKey) {
         return bindingNames;
     }
 
-    public static String replaceMustacheWithPlaceholder(String query, List<String> mustacheBindings) {
+    public static String replaceMustacheWithPlaceholder(String query, List<MustacheBindingToken> mustacheBindings) {
         return replaceMustacheUsingPatterns(query, APPSMITH_SUBSTITUTION_PLACEHOLDER, mustacheBindings, placeholderTrimmingPattern, APPSMITH_SUBSTITUTION_PLACEHOLDER);
     }
 
-    public static String replaceMustacheWithQuestionMark(String query, List<String> mustacheBindings) {
+    public static String replaceMustacheWithQuestionMark(String query, List<MustacheBindingToken> mustacheBindings) {
 
         return replaceMustacheUsingPatterns(query, "?", mustacheBindings, quoteQuestionPattern, postQuoteTrimmingQuestionMark);
     }
 
     private static String replaceMustacheUsingPatterns(String query,
                                                        String placeholder,
-                                                       List<String> mustacheBindings,
+                                                       List<MustacheBindingToken> mustacheBindings,
                                                        Pattern sanitizePattern,
                                                        String replacement) {
         ActionConfiguration actionConfiguration = new ActionConfiguration();
         actionConfiguration.setBody(query);
 
-        Set<String> mustacheSet = new HashSet<>(mustacheBindings);
+        Set<MustacheBindingToken> mustacheSet = new HashSet<>(mustacheBindings);
 
-        Map<String, String> replaceParamsMap = mustacheSet.stream().collect(Collectors.toMap(Function.identity(), v -> placeholder));
+        Map<String, String> replaceParamsMap = mustacheSet.stream().collect(Collectors.toMap(k -> k.getValue(), v -> placeholder));
 
         // Replace the mustaches with the values mapped to each mustache in replaceParamsMap
         ActionConfiguration updatedActionConfiguration = renderFieldValues(actionConfiguration, replaceParamsMap);

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/models/MustacheBindingToken.java

@@ -0,0 +1,21 @@
+package com.appsmith.external.models;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class MustacheBindingToken {
+
+    String value;
+    int startIndex;
+    boolean includesHandleBars = false;
+}