Add Open Parameters to Flat_object Field Type

kkewwei · May 28, 2024 · 25c607c · 25c607c
1 parent 1c0a274
commit 25c607c
Show file tree

Hide file tree

Showing 7 changed files with 625 additions and 53 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
 - Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
 - Adds support to inject telemetry instances to plugins ([#13636](https://github.com/opensearch-project/OpenSearch/pull/13636))
+- Add Open Parameters to Flat_object Field Type ([#13853](https://github.com/opensearch-project/OpenSearch/pull/13853))
 
 ### Deprecated
 

diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
@@ -18,6 +18,7 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentLocation;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.mapper.MapperParsingException;
 
 import java.io.IOException;
 import java.math.BigInteger;
@@ -44,6 +45,10 @@ public class JsonToStringXContentParser extends AbstractXContentParser {
 
     private DeprecationHandler deprecationHandler;
 
+    private int depthLimit;
+    private String nullValue;
+    private int ignoreAbove;
+
     private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath";
     private static final String VALUE_SUFFIX = "._value";
     private static final String DOT_SYMBOL = ".";
@@ -53,19 +58,25 @@ public JsonToStringXContentParser(
         NamedXContentRegistry xContentRegistry,
         DeprecationHandler deprecationHandler,
         XContentParser parser,
-        String fieldTypeName
+        String fieldTypeName,
+        int depthLimit,
+        String nullValue,
+        int ignoreAbove
     ) throws IOException {
         super(xContentRegistry, deprecationHandler);
         this.deprecationHandler = deprecationHandler;
         this.xContentRegistry = xContentRegistry;
         this.parser = parser;
         this.fieldTypeName = fieldTypeName;
+        this.depthLimit = depthLimit;
+        this.nullValue = nullValue;
+        this.ignoreAbove = ignoreAbove;
     }
 
     public XContentParser parseObject() throws IOException {
         builder.startObject();
         StringBuilder path = new StringBuilder(fieldTypeName);
-        parseToken(path, null);
+        parseToken(path, null, 1);
         builder.field(this.fieldTypeName, keyList);
         builder.field(this.fieldTypeName + VALUE_SUFFIX, valueList);
         builder.field(this.fieldTypeName + VALUE_AND_PATH_SUFFIX, valueAndPathList);
@@ -74,7 +85,15 @@ public XContentParser parseObject() throws IOException {
         return JsonXContent.jsonXContent.createParser(this.xContentRegistry, this.deprecationHandler, String.valueOf(jString));
     }
 
-    private void parseToken(StringBuilder path, String currentFieldName) throws IOException {
+    private void parseToken(StringBuilder path, String currentFieldName, int depth) throws IOException {
+        if (depth >= depthLimit) {
+            throw new MapperParsingException(
+                "the depth of flat_object field path [" + path + "] is bigger than maximum" + " depth [" + depthLimit + "]"
+            );
+        }
+        if (depth == 1 && processNoNestedValue()) {
+            return;
+        }
 
         while (this.parser.nextToken() != Token.END_OBJECT) {
             if (this.parser.currentName() != null) {
@@ -100,12 +119,12 @@ private void parseToken(StringBuilder path, String currentFieldName) throws IOEx
                     this.keyList.add(fieldNameSuffix);
                 }
             } else if (this.parser.currentToken() == Token.START_ARRAY) {
-                parseToken(path, currentFieldName);
+                parseToken(path, currentFieldName, depth);
                 break;
             } else if (this.parser.currentToken() == Token.END_ARRAY) {
                 // skip
             } else if (this.parser.currentToken() == Token.START_OBJECT) {
-                parseToken(path, currentFieldName);
+                parseToken(path, currentFieldName, depth + 1);
                 int dotIndex = path.lastIndexOf(DOT_SYMBOL, path.length());
 
                 if (dotIndex != -1 && path.length() > currentFieldName.length()) {
@@ -115,9 +134,10 @@ private void parseToken(StringBuilder path, String currentFieldName) throws IOEx
                 if (!path.toString().contains(currentFieldName)) {
                     path.append(DOT_SYMBOL).append(currentFieldName);
                 }
-                parseValue(parsedFields);
-                this.valueList.add(parsedFields.toString());
-                this.valueAndPathList.add(path + EQUAL_SYMBOL + parsedFields);
+                if (parseValue(parsedFields)) {
+                    this.valueList.add(parsedFields.toString());
+                    this.valueAndPathList.add(path + EQUAL_SYMBOL + parsedFields);
+                }
                 int dotIndex = path.lastIndexOf(DOT_SYMBOL, path.length());
                 if (dotIndex != -1 && path.length() > currentFieldName.length()) {
                     path.setLength(path.length() - currentFieldName.length() - 1);
@@ -127,13 +147,35 @@ private void parseToken(StringBuilder path, String currentFieldName) throws IOEx
         }
     }
 
-    private void parseValue(StringBuilder parsedFields) throws IOException {
+    private boolean processNoNestedValue() throws IOException {
+        if (parser.currentToken() == Token.VALUE_NULL) {
+            if (nullValue != null) {
+                this.valueList.add(nullValue);
+            }
+            return true;
+        } else if (this.parser.currentToken() == Token.VALUE_STRING
+            || this.parser.currentToken() == Token.VALUE_NUMBER
+            || this.parser.currentToken() == Token.VALUE_BOOLEAN) {
+                String value = this.parser.textOrNull();
+                if (value != null && value.length() <= ignoreAbove) {
+                    this.valueList.add(value);
+                }
+                return true;
+            }
+        return false;
+    }
+
+    private boolean parseValue(StringBuilder parsedFields) throws IOException {
         switch (this.parser.currentToken()) {
             case VALUE_BOOLEAN:
             case VALUE_NUMBER:
             case VALUE_STRING:
             case VALUE_NULL:
-                parsedFields.append(this.parser.textOrNull());
+                String value = this.parser.textOrNull();
+                if (value != null && value.length() <= ignoreAbove) {
+                    parsedFields.append(value);
+                    return true;
+                }
                 break;
             // Handle other token types as needed
             case FIELD_NAME:
@@ -144,6 +186,7 @@ private void parseValue(StringBuilder parsedFields) throws IOException {
             default:
                 throw new IOException("Unsupported token type [" + parser.currentToken() + "]");
         }
+        return false;
     }
 
     @Override